MC_MTA_Clean.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "from copy import copy\n",
    "import numpy as np \n",
    "from numpy import exp, log\n",
    "import pandas as pd \n",
    "import os\n",
    "\n",
    "import tensorflow as tf\n",
    "import keras\n",
    "from keras.layers import Embedding, Dense, Concatenate, Flatten, Input, Reshape, AdditiveAttention, LSTM, Bidirectional\n",
    "from keras.models import Sequential\n",
    "from keras.callbacks import EarlyStopping\n",
    "from keras.utils import to_categorical\n",
    "from keras.losses import binary_crossentropy\n",
    "from keras.optimizers import Adam\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score,confusion_matrix,classification_report,f1_score\n",
    "import matplotlib.pyplot as plt\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "#pd.set_option('display.max_colwidth',100)\n",
    "\n",
    "\n",
    "pd.set_option('display.max_colwidth', -1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Simulation Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Simulate inputs for 3 channels, constant 2 time steps, 100K observations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "channels = ['facebook', 'google', 'snapchat']\n",
    "num_chnls = len(channels)\n",
    "N = 100000\n",
    "\n",
    "np.random.seed(498)\n",
    "df = pd.DataFrame({'t1':np.random.uniform(-.5,num_chnls-.5,N)\n",
    "                   , 't2':np.random.uniform(-.5,num_chnls-.5,N)\n",
    "                   , 't3':np.random.uniform(-.5,num_chnls-.5,N)})\n",
    "\n",
    "df['t1'] = df['t1'].apply(lambda x: int(round(x)))\n",
    "df['t2'] = df['t2'].apply(lambda x: int(round(x)))\n",
    "df['t3'] = df['t3'].apply(lambda x: int(round(x)))\n",
    "\n",
    "#np.random.seed(1)\n",
    "df['gap1'] = np.random.gamma(1.2, 1.2, N)\n",
    "#np.random.seed(2)\n",
    "df['gap2'] = np.random.gamma(1.2, 1.2, N)\n",
    "df['gap1'] = df['gap1'] + df['gap2']\n",
    "#np.random.seed(3)\n",
    "df['gender'] = np.random.binomial(1, .4, N)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([684., 142.,  87.,  40.,  17.,  13.,   5.,   6.,   3.,   3.]),\n",
       " array([6.62376023e-08, 2.34107815e-01, 4.68215563e-01, 7.02323311e-01,\n",
       "        9.36431060e-01, 1.17053881e+00, 1.40464656e+00, 1.63875430e+00,\n",
       "        1.87286205e+00, 2.10696980e+00, 2.34107755e+00]),\n",
       " <a list of 10 Patch objects>)"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAQoklEQVR4nO3db4xldX3H8fenLGjrv+XPQDe7W1fjxmqTAuuEriEx1q2NYMOSFBJMIyvZZpuWthqbtFsf1LTpA3wiLW2D2YrtYqxCUctW0ZYuGNMHUAdEBFfLSJSd7JYd+bNoqRrabx/Mb2HYvbNzd+fOjPPb9yu5Oed8z+/O/c7h7mfO/ObcQ6oKSVJffmq5G5AkjZ7hLkkdMtwlqUOGuyR1yHCXpA6tWu4GAM4555zasGHDcrchSSvKfffd972qGhu07yci3Dds2MDExMRytyFJK0qS7861z2kZSerQvOGe5PVJHpj1eCbJ+5KcleTOJI+05ZltfJLckGQyyYNJNi3+tyFJmm3ecK+qb1XVBVV1AfAm4Fngs8BOYG9VbQT2tm2AS4CN7bEDuHExGpckze1Ep2W2AN+uqu8CW4Hdrb4buLytbwVurhn3AKuTrBlJt5KkoZxouF8FfLKtn1dVBwHa8txWXwvsn/WcqVZ7kSQ7kkwkmZienj7BNiRJxzN0uCc5A7gM+Mf5hg6oHXN3sqraVVXjVTU+NjbwSh5J0kk6kTP3S4D7q+rxtv34kemWtjzU6lPA+lnPWwccWGijkqThnUi4v4sXpmQA9gDb2vo24PZZ9avbVTObgcNHpm8kSUtjqA8xJfkZ4O3Ab80qXwfcmmQ78BhwZavfAVwKTDJzZc01I+tWkjSUocK9qp4Fzj6q9gQzV88cPbaAa0fS3RA27Pz8Ur3UMb5z3TuX7bUl6Xj8hKokdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtSh4YK9ySrk9yW5JtJ9iV5c5KzktyZ5JG2PLONTZIbkkwmeTDJpsX9FiRJRxv2zP0vgS9W1c8D5wP7gJ3A3qraCOxt2wCXABvbYwdw40g7liTNa95wT/JK4C3ATQBV9eOqehrYCuxuw3YDl7f1rcDNNeMeYHWSNSPvXJI0p2HO3F8LTAN/l+SrST6a5GXAeVV1EKAtz23j1wL7Zz1/qtVeJMmOJBNJJqanpxf0TUiSXmyYcF8FbAJurKoLgf/mhSmYQTKgVscUqnZV1XhVjY+NjQ3VrCRpOMOE+xQwVVX3tu3bmAn7x49Mt7TloVnj1896/jrgwGjalSQNY95wr6r/AvYneX0rbQG+AewBtrXaNuD2tr4HuLpdNbMZOHxk+kaStDRWDTnu94BPJDkDeBS4hpkfDLcm2Q48BlzZxt4BXApMAs+2sZKkJTRUuFfVA8D4gF1bBowt4NoF9iVJWgA/oSpJHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUoaHCPcl3knw9yQNJJlrtrCR3JnmkLc9s9SS5IclkkgeTbFrMb0CSdKwTOXP/5aq6oKrG2/ZOYG9VbQT2tm2AS4CN7bEDuHFUzUqShrOQaZmtwO62vhu4fFb95ppxD7A6yZoFvI4k6QQNG+4F/GuS+5LsaLXzquogQFue2+prgf2znjvVai+SZEeSiSQT09PTJ9e9JGmgVUOOu7iqDiQ5F7gzyTePMzYDanVMoWoXsAtgfHz8mP2SpJM31Jl7VR1oy0PAZ4GLgMePTLe05aE2fApYP+vp64ADo2pYkjS/ecM9ycuSvOLIOvCrwEPAHmBbG7YNuL2t7wGublfNbAYOH5m+kSQtjWGmZc4DPpvkyPh/qKovJvkKcGuS7cBjwJVt/B3ApcAk8Cxwzci7liQd17zhXlWPAucPqD8BbBlQL+DakXQnSTopfkJVkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1aOhwT3Jakq8m+Vzbfk2Se5M8kuSWJGe0+kva9mTbv2FxWpckzeVEztzfC+ybtf0h4Pqq2gg8BWxv9e3AU1X1OuD6Nk6StISGCvck64B3Ah9t2wHeBtzWhuwGLm/rW9s2bf+WNl6StESGPXP/C+APgf9r22cDT1fVc217Cljb1tcC+wHa/sNt/Isk2ZFkIsnE9PT0SbYvSRpk3nBP8mvAoaq6b3Z5wNAaYt8LhapdVTVeVeNjY2NDNStJGs6qIcZcDFyW5FLgpcArmTmTX51kVTs7XwccaOOngPXAVJJVwKuAJ0feuSRpTvOeuVfVH1fVuqraAFwF3FVVvwHcDVzRhm0Dbm/re9o2bf9dVXXMmbskafEs5Dr3PwLen2SSmTn1m1r9JuDsVn8/sHNhLUqSTtQw0zLPq6ovAV9q648CFw0Y80PgyhH0Jkk6SX5CVZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalD84Z7kpcm+Y8kX0vycJI/bfXXJLk3ySNJbklyRqu/pG1Ptv0bFvdbkCQdbZgz9x8Bb6uq84ELgHck2Qx8CLi+qjYCTwHb2/jtwFNV9Trg+jZOkrSE5g33mvGDtnl6exTwNuC2Vt8NXN7Wt7Zt2v4tSTKyjiVJ8xpqzj3JaUkeAA4BdwLfBp6uqufakClgbVtfC+wHaPsPA2cP+Jo7kkwkmZienl7YdyFJepGhwr2q/reqLgDWARcBbxg0rC0HnaXXMYWqXVU1XlXjY2Njw/YrSRrCCV0tU1VPA18CNgOrk6xqu9YBB9r6FLAeoO1/FfDkKJqVJA1nmKtlxpKsbus/DfwKsA+4G7iiDdsG3N7W97Rt2v67quqYM3dJ0uJZNf8Q1gC7k5zGzA+DW6vqc0m+AXwqyZ8DXwVuauNvAj6eZJKZM/arFqFvSdJxzBvuVfUgcOGA+qPMzL8fXf8hcOVIupMknRQ/oSpJHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUoXnDPcn6JHcn2Zfk4STvbfWzktyZ5JG2PLPVk+SGJJNJHkyyabG/CUnSiw1z5v4c8AdV9QZgM3BtkjcCO4G9VbUR2Nu2AS4BNrbHDuDGkXctSTquecO9qg5W1f1t/fvAPmAtsBXY3YbtBi5v61uBm2vGPcDqJGtG3rkkaU4nNOeeZANwIXAvcF5VHYSZHwDAuW3YWmD/rKdNtdrRX2tHkokkE9PT0yfeuSRpTkOHe5KXA58G3ldVzxxv6IBaHVOo2lVV41U1PjY2NmwbkqQhDBXuSU5nJtg/UVWfaeXHj0y3tOWhVp8C1s96+jrgwGjalSQNY5irZQLcBOyrqg/P2rUH2NbWtwG3z6pf3a6a2QwcPjJ9I0laGquGGHMx8G7g60keaLUPANcBtybZDjwGXNn23QFcCkwCzwLXjLRjSdK85g33qvp3Bs+jA2wZML6AaxfYlyRpAfyEqiR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6NMxdITWHDTs/vyyv+53r3rksrytp5fDMXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktShecM9yceSHEry0KzaWUnuTPJIW57Z6klyQ5LJJA8m2bSYzUuSBhvmzP3vgXccVdsJ7K2qjcDetg1wCbCxPXYAN46mTUnSiZg33Kvqy8CTR5W3Arvb+m7g8ln1m2vGPcDqJGtG1awkaTgnO+d+XlUdBGjLc1t9LbB/1ripVpMkLaFR/0E1A2o1cGCyI8lEkonp6ekRtyFJp7aTDffHj0y3tOWhVp8C1s8atw44MOgLVNWuqhqvqvGxsbGTbEOSNMjJhvseYFtb3wbcPqt+dbtqZjNw+Mj0jSRp6cx7P/cknwTeCpyTZAr4IHAdcGuS7cBjwJVt+B3ApcAk8CxwzSL0fMpbrvvIg/eSl1aKecO9qt41x64tA8YWcO1Cm5IkLYyfUJWkDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SerQvHeFlGZbrtsNe6th6cR45i5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI65KWQWhG8BFM6MYa7dBzL9UMF/MGihVmUaZkk70jyrSSTSXYuxmtIkuY28jP3JKcBfwO8HZgCvpJkT1V9Y9SvJfVsOX9rWC7+tjI6izEtcxEwWVWPAiT5FLAVMNwlHZc/0EZnMcJ9LbB/1vYU8EtHD0qyA9jRNn+Q5Fsn+XrnAN87yef2wmPgMQCPAazAY5APLejpr55rx2KEewbU6phC1S5g14JfLJmoqvGFfp2VzGPgMQCPAXgMZluMP6hOAetnba8DDizC60iS5rAY4f4VYGOS1yQ5A7gK2LMIryNJmsPIp2Wq6rkkvwv8C3Aa8LGqenjUrzPLgqd2OuAx8BiAxwA8Bs9L1THT4ZKkFc57y0hShwx3SerQign3+W5pkOQlSW5p++9NsmHpu1xcQxyD9ySZTvJAe/zmcvS5WJJ8LMmhJA/NsT9JbmjH58Ekm5a6x8U2xDF4a5LDs94Df7LUPS62JOuT3J1kX5KHk7x3wJju3wvzqqqf+Aczf5j9NvBa4Azga8AbjxrzO8BH2vpVwC3L3fcyHIP3AH+93L0u4jF4C7AJeGiO/ZcCX2DmsxabgXuXu+dlOAZvBT633H0u8jFYA2xq668A/nPAv4Xu3wvzPVbKmfvztzSoqh8DR25pMNtWYHdbvw3YkmTQB6pWqmGOQdeq6svAk8cZshW4uWbcA6xOsmZpulsaQxyD7lXVwaq6v61/H9jHzCfjZ+v+vTCflRLug25pcPR/zOfHVNVzwGHg7CXpbmkMcwwAfr39GnpbkvUD9vds2GPUuzcn+VqSLyT5heVuZjG16dcLgXuP2nXKvxdWSrgPc0uDoW57sIIN8/39M7Chqn4R+Dde+E3mVNH7e2AY9wOvrqrzgb8C/mmZ+1k0SV4OfBp4X1U9c/TuAU85pd4LKyXch7mlwfNjkqwCXkVfv77Oewyq6omq+lHb/FvgTUvU20+KU/7WF1X1TFX9oK3fAZye5JxlbmvkkpzOTLB/oqo+M2DIKf9eWCnhPswtDfYA29r6FcBd1f6y0ol5j8FRc4qXMTMXeSrZA1zdrpTYDByuqoPL3dRSSvKzR/7WlOQiZv6NP7G8XY1W+/5uAvZV1YfnGHbKvxdWxP9mr+a4pUGSPwMmqmoPM/+xP55kkpkz9quWr+PRG/IY/H6Sy4DnmDkG71m2hhdBkk8yczXIOUmmgA8CpwNU1UeAO5i5SmISeBa4Znk6XTxDHIMrgN9O8hzwP8BVnZ3kAFwMvBv4epIHWu0DwM/BqfNemI+3H5CkDq2UaRlJ0gkw3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KH/h+sCGrWv4gtVwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(np.random.gamma(.5, .5, 1000))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAD6CAYAAABDPiuvAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAV6UlEQVR4nO3dfYxdd53f8fdn7SSwPGwcYqhle3HYWl0cVEyYJi6s2hBQ4mS1dVBBcko3hqbyQpMK1NWKsEgNT1Hhj91UUSFVWFyciuKkARo3cup1QxCikIcJGCcmBA8OJV5byYCTkAg1NNG3f9zfbA+TOzN3Hu6Mid8v6eie+z2/37m/e+Z4PnMe7nWqCknSye23lnoAkqSlZxhIkgwDSZJhIEnCMJAkYRhIkhggDJK8JMm9Sb6f5GCSj7f6F5M8kmR/mza2epJcn2QsyYEk53TWtS3JoTZt69TfnOSB1uf6JBnGm5Uk9bd8gDbPAhdU1TNJTgG+leSOtuzPqurWSe0vBta36TzgBuC8JGcA1wAjQAH3J9ldVU+0NtuBu4E9wGbgDqZx5pln1rp16wYYviRpwv333/+zqlo5uT5jGFTvU2nPtKentGm6T6ptAW5q/e5OcnqSVcD5wL6qOg6QZB+wOck3gFdW1Xda/SbgUmYIg3Xr1jE6OjrT8CVJHUn+d7/6QNcMkixLsh94nN4v9HvaomvbqaDrkpzWaquBRzvdj7TadPUjfer9xrE9yWiS0fHx8UGGLkkawEBhUFXPV9VGYA1wbpI3AB8Bfh/4B8AZwIdb837n+2sO9X7juLGqRqpqZOXKFxzlSJLmaFZ3E1XVk8A3gM1Vdax6ngX+E3Bua3YEWNvptgY4OkN9TZ+6JGmRDHI30cokp7f5lwLvAH7YrgPQ7vy5FHiwddkNXN7uKtoEPFVVx4C9wIVJViRZAVwI7G3Lnk6yqa3rcuC2hX2bkqTpDHI30SpgZ5Jl9MLjlqq6PcnXk6ykd5pnP/D+1n4PcAkwBvwSeB9AVR1P8kngvtbuExMXk4EPAF8EXkrvwvG0F48lSQsrv6lfYT0yMlLeTSRJs5Pk/qoamVz3E8iSJMNAkmQYSJIY7ALyi04+PthXH9X6PsV/9pt5jUWSpnNShoEkzdd8vk6zvvTCzjk0YN9rhvMHqWEwS/PaATyoOCks5Xfuuo9prrxmIEnyyEBacB+b+zWpvMc/7bU0PDKQJBkGkiTDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSGCAMkrwkyb1Jvp/kYJKPt/pZSe5JcijJzUlObfXT2vOxtnxdZ10fafWHk1zUqW9utbEkVy/825QkTWeQI4NngQuq6o3ARmBzkk3AZ4Drqmo98ARwRWt/BfBEVf1d4LrWjiQbgK3A2cBm4HNJliVZBnwWuBjYAFzW2kqSFsmMYVA9z7Snp7SpgAuAW1t9J3Bpm9/SntOWvz1JWn1XVT1bVY8AY8C5bRqrqsNV9StgV2srSVokA10zaH/B7wceB/YBPwaerKrnWpMjwOo2vxp4FKAtfwp4Vbc+qc9U9X7j2J5kNMno+Pj4IEOXJA1goDCoqueraiOwht5f8q/v16w99vs//2oO9X7juLGqRqpqZOXKlTMPXJI0kFndTVRVTwLfADYBpyeZ+D+U1wBH2/wRYC1AW/47wPFufVKfqeqSpEUyyN1EK5Oc3uZfCrwDeAi4C3hXa7YNuK3N727Pacu/XlXV6lvb3UZnAeuBe4H7gPXt7qRT6V1k3r0Qb06SNJjlMzdhFbCz3fXzW8AtVXV7kh8Au5J8Cvge8IXW/gvAf04yRu+IYCtAVR1McgvwA+A54Mqqeh4gyVXAXmAZsKOqDi7YO5QkzWjGMKiqA8Cb+tQP07t+MLn+f4B3T7Gua4Fr+9T3AHsGGK8kaQj8BLIkyTCQJBkGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CSxABhkGRtkruSPJTkYJIPtvrHkvxNkv1tuqTT5yNJxpI8nOSiTn1zq40lubpTPyvJPUkOJbk5yakL/UYlSVMb5MjgOeBPq+r1wCbgyiQb2rLrqmpjm/YAtGVbgbOBzcDnkixLsgz4LHAxsAG4rLOez7R1rQeeAK5YoPcnSRrAjGFQVceq6rtt/mngIWD1NF22ALuq6tmqegQYA85t01hVHa6qXwG7gC1JAlwA3Nr67wQunesbkiTN3qyuGSRZB7wJuKeVrkpyIMmOJCtabTXwaKfbkVabqv4q4Mmqem5Svd/rb08ymmR0fHx8NkOXJE1j4DBI8nLgK8CHquoXwA3A7wEbgWPAX0w07dO95lB/YbHqxqoaqaqRlStXDjp0SdIMlg/SKMkp9ILgS1X1VYCqeqyz/PPA7e3pEWBtp/sa4Gib71f/GXB6kuXt6KDbXpK0CAa5myjAF4CHquovO/VVnWbvBB5s87uBrUlOS3IWsB64F7gPWN/uHDqV3kXm3VVVwF3Au1r/bcBt83tbkqTZGOTI4K3AHwMPJNnfan9O726gjfRO6fwE+BOAqjqY5BbgB/TuRLqyqp4HSHIVsBdYBuyoqoNtfR8GdiX5FPA9euEjSVokM4ZBVX2L/uf190zT51rg2j71Pf36VdVhencbSZKWgJ9AliQZBpIkw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIYIAySrE1yV5KHkhxM8sFWPyPJviSH2uOKVk+S65OMJTmQ5JzOura19oeSbOvU35zkgdbn+iQZxpuVJPU3yJHBc8CfVtXrgU3AlUk2AFcDd1bVeuDO9hzgYmB9m7YDN0AvPIBrgPOAc4FrJgKktdne6bd5/m9NkjSoGcOgqo5V1Xfb/NPAQ8BqYAuwszXbCVza5rcAN1XP3cDpSVYBFwH7qup4VT0B7AM2t2WvrKrvVFUBN3XWJUlaBLO6ZpBkHfAm4B7gNVV1DHqBAby6NVsNPNrpdqTVpqsf6VPv9/rbk4wmGR0fH5/N0CVJ0xg4DJK8HPgK8KGq+sV0TfvUag71FxarbqyqkaoaWbly5UxDliQNaKAwSHIKvSD4UlV9tZUfa6d4aI+Pt/oRYG2n+xrg6Az1NX3qkqRFMsjdRAG+ADxUVX/ZWbQbmLgjaBtwW6d+eburaBPwVDuNtBe4MMmKduH4QmBvW/Z0kk3ttS7vrEuStAiWD9DmrcAfAw8k2d9qfw58GrglyRXAT4F3t2V7gEuAMeCXwPsAqup4kk8C97V2n6iq423+A8AXgZcCd7RJkrRIZgyDqvoW/c/rA7y9T/sCrpxiXTuAHX3qo8AbZhqLJGk4/ASyJMkwkCQZBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkiQHCIMmOJI8nebBT+1iSv0myv02XdJZ9JMlYkoeTXNSpb261sSRXd+pnJbknyaEkNyc5dSHfoCRpZoMcGXwR2Nynfl1VbWzTHoAkG4CtwNmtz+eSLEuyDPgscDGwAbistQX4TFvXeuAJ4Ir5vCFJ0uzNGAZV9U3g+IDr2wLsqqpnq+oRYAw4t01jVXW4qn4F7AK2JAlwAXBr678TuHSW70GSNE/zuWZwVZID7TTSilZbDTzaaXOk1aaqvwp4sqqem1TvK8n2JKNJRsfHx+cxdElS11zD4Abg94CNwDHgL1o9fdrWHOp9VdWNVTVSVSMrV66c3YglSVNaPpdOVfXYxHySzwO3t6dHgLWdpmuAo22+X/1nwOlJlrejg257SdIimdORQZJVnafvBCbuNNoNbE1yWpKzgPXAvcB9wPp259Cp9C4y766qAu4C3tX6bwNum8uYJElzN+ORQZIvA+cDZyY5AlwDnJ9kI71TOj8B/gSgqg4muQX4AfAccGVVPd/WcxWwF1gG7Kiqg+0lPgzsSvIp4HvAFxbs3UmSBjJjGFTVZX3KU/7CrqprgWv71PcAe/rUD9O720iStET8BLIkyTCQJBkGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CSxABhkGRHkseTPNipnZFkX5JD7XFFqyfJ9UnGkhxIck6nz7bW/lCSbZ36m5M80PpcnyQL/SYlSdMb5Mjgi8DmSbWrgTuraj1wZ3sOcDGwvk3bgRugFx7ANcB5wLnANRMB0tps7/Sb/FqSpCGbMQyq6pvA8UnlLcDONr8TuLRTv6l67gZOT7IKuAjYV1XHq+oJYB+wuS17ZVV9p6oKuKmzLknSIpnrNYPXVNUxgPb46lZfDTzaaXek1aarH+lT7yvJ9iSjSUbHx8fnOHRJ0mQLfQG53/n+mkO9r6q6sapGqmpk5cqVcxyiJGmyuYbBY+0UD+3x8VY/AqzttFsDHJ2hvqZPXZK0iOYaBruBiTuCtgG3deqXt7uKNgFPtdNIe4ELk6xoF44vBPa2ZU8n2dTuIrq8sy5J0iJZPlODJF8GzgfOTHKE3l1BnwZuSXIF8FPg3a35HuASYAz4JfA+gKo6nuSTwH2t3SeqauKi9Afo3bH0UuCONkmSFtGMYVBVl02x6O192hZw5RTr2QHs6FMfBd4w0zgkScPjJ5AlSYaBJMkwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJOYZBkl+kuSBJPuTjLbaGUn2JTnUHle0epJcn2QsyYEk53TWs621P5Rk2/zekiRpthbiyOBtVbWxqkba86uBO6tqPXBnew5wMbC+TduBG6AXHsA1wHnAucA1EwEiSVocwzhNtAXY2eZ3Apd26jdVz93A6UlWARcB+6rqeFU9AewDNg9hXJKkKcw3DAr46yT3J9neaq+pqmMA7fHVrb4aeLTT90irTVV/gSTbk4wmGR0fH5/n0CVJE5bPs/9bq+poklcD+5L8cJq26VOraeovLFbdCNwIMDIy0reNJGn25nVkUFVH2+PjwNfonfN/rJ3+oT0+3pofAdZ2uq8Bjk5TlyQtkjmHQZKXJXnFxDxwIfAgsBuYuCNoG3Bbm98NXN7uKtoEPNVOI+0FLkyyol04vrDVJEmLZD6niV4DfC3JxHr+S1X9jyT3AbckuQL4KfDu1n4PcAkwBvwSeB9AVR1P8kngvtbuE1V1fB7jkiTN0pzDoKoOA2/sU/858PY+9QKunGJdO4Adcx2LJGl+/ASyJMkwkCQZBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAksQJFAZJNid5OMlYkquXejySdDI5IcIgyTLgs8DFwAbgsiQblnZUknTyOCHCADgXGKuqw1X1K2AXsGWJxyRJJ43lSz2AZjXwaOf5EeC8yY2SbAe2t6fPJHl4jq93JvCzmRplFtVBZOauA41rCTiu2VmS/Qtm3Md+o7fXEhjauPKeefT9WOY7rtf2K54oYdBvF64XFKpuBG6c94slo1U1Mt/1LDTHNTuOa3Yc1+ycbOM6UU4THQHWdp6vAY4u0Vgk6aRzooTBfcD6JGclORXYCuxe4jFJ0knjhDhNVFXPJbkK2AssA3ZU1cEhvuS8TzUNieOaHcc1O45rdk6qcaXqBafmJUknmRPlNJEkaQkZBpKkF18YzPS1FklOS3JzW35PknWdZR9p9YeTXLSIY/o3SX6Q5ECSO5O8trPs+ST727TgF9UHGNt7k4x3xvAvO8u2JTnUpm2LPK7rOmP6UZInO8uGss2S7EjyeJIHp1ieJNe3MR9Ick5n2TC31Uzjek8bz4Ek307yxs6ynyR5oG2r0UUe1/lJnur8rP5tZ9nQvp5mgHH9WWdMD7b96Yy2bJjba22Su5I8lORgkg/2aTO8fayqXjQTvYvPPwZeB5wKfB/YMKnNvwL+Y5vfCtzc5je09qcBZ7X1LFukMb0N+O02/4GJMbXnzyzx9nov8B/69D0DONweV7T5FYs1rknt/zW9mw6Gus2AfwScAzw4xfJLgDvofW5mE3DPsLfVgON6y8Tr0fvKl3s6y34CnLlE2+t84Pb5/vwXelyT2v4R8PVF2l6rgHPa/CuAH/X59zi0fezFdmQwyNdabAF2tvlbgbcnSavvqqpnq+oRYKytb+hjqqq7quqX7end9D5nsRjm8zUgFwH7qup4VT0B7AM2L9G4LgO+vECvPaWq+iZwfJomW4Cbqudu4PQkqxjutppxXFX17fa6sIj71wDbaypD/XqaWY5rUfYtgKo6VlXfbfNPAw/R+3aGrqHtYy+2MOj3tRaTN+bftqmq54CngFcN2HdYY+q6gl7yT3hJktEkdye5dAHGM5ex/dN2SHprkokPBw5re81q3e2U2lnA1zvlYW6z6Uw17mFuq9mavH8V8NdJ7k/v614W2z9M8v0kdyQ5u9VOiO2V5Lfp/UL9Sqe8KNsrvdPXbwLumbRoaPvYCfE5gwU0yNdaTNVmoK/EmIOB15vknwMjwD/ulH+3qo4meR3w9SQPVNWPF2Bcg47tvwNfrqpnk7yf3lHVBQP2Hea4JmwFbq2q5zu1YW6z6Sz2vjUrSd5GLwz+oFN+a9tWrwb2Jflh+8t5MXwXeG1VPZPkEuC/Aes5QbYXvVNE/6uqukcRQ99eSV5OL4A+VFW/mLy4T5cF2cdebEcGg3ytxd+2SbIc+B16h4zD+kqMgdab5B3AR4F/UlXPTtSr6mh7PAx8g95fCwtlxrFV1c874/k88OZB+w5zXB1bmXQYP+RtNp2pxr3kX7eS5O8DfwVsqaqfT9Q72+px4GsszKnRgVTVL6rqmTa/BzglyZmcANurmW7fGsr2SnIKvSD4UlV9tU+T4e1jw7gQslQTvSOdw/ROG0xceDp7Upsr+fULyLe0+bP59QvIh1mYC8iDjOlN9C6YrZ9UXwGc1ubPBA6xsBfSBhnbqs78O4G76/9fsHqkjXFFmz9jscbV2v09ehf0sojbbB1TXxD9Q3794t69w95WA47rd+ldA3vLpPrLgFd05r8NbF7Ecf2diZ8dvV+qP23bbqCf/7DG1ZZP/JH4ssXaXu293wT8+2naDG0fW7CNe6JM9K62/4jeL9ePtton6P3FDfAS4L+2fxz3Aq/r9P1o6/cwcPEijul/Ao8B+9u0u9XfAjzQ/jE8AFyxBNvr3wEH2xjuAn6/0/dftO04BrxvMcfVnn8M+PSkfkPbZvT+SjwG/F96f4ldAbwfeH9bHnr/SdOP22uPLNK2mmlcfwU80dm/Rlv9dW07fb/9jD+6yOO6qrNv3U0nrPr9/BdrXK3Ne+ndUNLtN+zt9Qf0Tu0c6PysLlmsfcyvo5AkveiuGUiS5sAwkCQZBpIkw0CShGEgScIwkCRhGEiSgP8HebLeh0cutyAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "_=plt.hist(df['t1'], bins=10, color='blue')\n",
    "_=plt.hist(df['t2'], bins=20, color='orange')\n",
    "_=plt.hist(df['t2'], bins=30, color='green')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Parameter Assignment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "v0 = -3 #intercept, negative since most people don't convert\n",
    "beta_g0 = 1.7 #google effect when it is first time step is stronger than otherwise\n",
    "beta_g1 = .8 #google effect when it is second time step\n",
    "beta_f0 = .9 #facebook effect when it is first time step \n",
    "beta_f1 = 1.5 #facebook effect when it is second time step\n",
    "beta_s0 = .7 #snapchat effect when it is first time step\n",
    "beta_s1 = .5 #snapchat effect when it is first time step\n",
    "beta_last_tch = 1 #effect boosted when channel is last touched, constant for all channels\n",
    "beta_gender = .4 #effect for gender \n",
    "alpha = -1.2 #decay effect for gap between time steps, constant for all channels\n",
    "c = 1 #constant to prevent exploding gap terms when they are <1\n",
    "\n",
    "#Error term\n",
    "#np.random.seed(654)\n",
    "epsilon = np.random.normal(0, 1, len(df))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "v0 = -abs(np.random.normal(3, .5, 1))[0] #intercept, negative since most people don't convert\n",
    "beta_g0 = abs(np.random.normal(1, .3, 1))[0] #google effect when it is first time step is stronger than otherwise\n",
    "beta_g1 = abs(np.random.normal(.8, .4, 1))[0] #google effect when it is second time step\n",
    "beta_f0 = abs(np.random.normal(1, .5, 1))[0] #facebook effect when it is first time step \n",
    "beta_f1 = abs(np.random.normal(1, .5, 1))[0] #facebook effect when it is second time step\n",
    "beta_s0 = abs(np.random.normal(1, .6, 1))[0] #snapchat effect when it is first time step\n",
    "beta_s1 = abs(np.random.normal(1, .6, 1))[0] #snapchat effect when it is first time step\n",
    "beta_last_tch = abs(np.random.normal(1, .1, 1))[0] #effect boosted when channel is last touched, constant for all channels\n",
    "beta_gender = abs(np.random.normal(.5, .2, 1))[0] #effect for gender \n",
    "alpha = -abs(np.random.normal(1, 1, 1))[0] #decay effect for gap between time steps, constant for all channels\n",
    "c = 1 #constant to prevent exploding gap terms when they are <1\n",
    "\n",
    "#Error term\n",
    "#np.random.seed(654)\n",
    "epsilon = np.random.normal(0, 1, len(df))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "v0: -2.606\n",
      "beta_g: [0.872, 1.41]\n",
      "beta_f: [0.949, 1.504]\n",
      "beta_s: [0.948, 1.234]\n",
      "beta_last_tch: 0.967\n",
      "beta_gender: 0.495\n",
      "alpha: -0.996\n"
     ]
    }
   ],
   "source": [
    "print('v0:', round(v0, 3))\n",
    "print('beta_g:', [round(beta_g0, 3), round(beta_g1, 3)])\n",
    "print('beta_f:', [round(beta_f0, 3), round(beta_f1, 3)])\n",
    "print('beta_s:', [round(beta_s0, 3), round(beta_s1, 3)])\n",
    "print('beta_last_tch:', round(beta_last_tch, 3))\n",
    "print('beta_gender:', round(beta_gender, 3))\n",
    "print('alpha:', round(alpha, 3))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "### Logistic Data Generating Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD4CAYAAAAAczaOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAATbElEQVR4nO3db4ydZX7e8e8VWLJtko1NGCiyrZoqVrKkaVg6Mq6Q2nRJjCGrNS+CxJYGi1hyX9BqV0qUsskLq5BIG1UKKWpDZS20JqEh7iYrrBUNmXg3ivoClvHCkoCX2tkkeGKCJ7KXTYq6EZtfX8w9u8f4zMwZe+b8mef7kY7OeX7Pfc7cz7HnOvfcz5+TqkKS1A3fMeoOSJKGx9CXpA4x9CWpQwx9SeoQQ1+SOuTKUXdgOddcc01t37591N2QpIly/Pjxv6qqqX7rxjr0t2/fzuzs7Ki7IUkTJcmfL7XO6R1J6hBDX5I6xNCXpA4x9CWpQwx9SeqQFUM/yQ8kebnn9vUkn0hydZKZJCfb/ebWPkkeTXIqyStJbu55rX2t/ckk+9ZzwyRJF1sx9Kvq9aq6qapuAv4p8A7wWeBB4FhV7QCOtWWAO4Ad7XYAeAwgydXAQeAWYCdwcPGDQpI0HKud3rkN+JOq+nNgL3C41Q8Dd7XHe4Ena8HzwKYk1wO3AzNVda6qzgMzwJ7L3gJJ0sBWG/r3AL/ZHl9XVW8CtPtrW30LcLrnOXOttlT9AkkOJJlNMjs/P7/K7kmSljNw6Ce5Cvgo8D9XatqnVsvULyxUHaqq6aqanprqexaxtC6S1d2kSbSakf4dwJeq6q22/FabtqHdn231OWBbz/O2AmeWqUuShmQ1of8xvj21A3AUWDwCZx/wTE/9vnYUzy7g7Tb98xywO8nmtgN3d6tJkoZkoAuuJfn7wI8D/6an/CngSJL9wBvA3a3+LHAncIqFI33uB6iqc0keBl5s7R6qqnOXvQWSpIFlnL8YfXp6urzKpoZltfP0Y/yro45Lcryqpvut84xcSeqQsb6evrTWPOpGXedIX5I6xNCXpA5xeke6REtNFbmDV+PMkb4kdYihL0kdYuhLUocY+pLUIYa+JHWIoS9JHWLoS1KHeJy+tMY8fl/jzJG+JHWII31tSF5YTerPkb4kdYihL0kdYuhLUocY+pLUIYa+JHXIQKGfZFOSzyT5SpITSf5ZkquTzCQ52e43t7ZJ8miSU0leSXJzz+vsa+1PJtm3XhslSepv0JH+fwJ+t6p+EPgR4ATwIHCsqnYAx9oywB3AjnY7ADwGkORq4CBwC7ATOLj4QSFJGo4VQz/JB4B/DjwOUFV/W1VfA/YCh1uzw8Bd7fFe4Mla8DywKcn1wO3ATFWdq6rzwAywZ023RpK0rEFG+v8ImAf+W5KXknw6yXcB11XVmwDt/trWfgtwuuf5c622VP0CSQ4kmU0yOz8/v+oNkiQtbZDQvxK4GXisqj4E/F++PZXTT79zIWuZ+oWFqkNVNV1V01NTUwN0T5I0qEFCfw6Yq6oX2vJnWPgQeKtN29Duz/a039bz/K3AmWXqkqQhWTH0q+ovgdNJfqCVbgNeA44Ci0fg7AOeaY+PAve1o3h2AW+36Z/ngN1JNrcduLtbTZI0JINecO3fAU8luQr4KnA/Cx8YR5LsB94A7m5tnwXuBE4B77S2VNW5JA8DL7Z2D1XVuTXZCknSQFJjfJHv6enpmp2dHXU3NIEm6SqbY/wrqAmV5HhVTfdb5xm5ktQhhr4kdYhfoqKJNknTONI4cKQvSR1i6EtShxj6ktQhhr4kdYihL0kdYuhLUocY+pLUIYa+JHWIoS9JHWLoS1KHGPqS1CGGviR1iKEvSR1i6EtShxj6ktQhhr4kdYihL0kdYuhLUocMFPpJ/izJHyV5Oclsq12dZCbJyXa/udWT5NEkp5K8kuTmntfZ19qfTLJvfTZJmixJ/5u0HlYz0v+XVXVTVU235QeBY1W1AzjWlgHuAHa02wHgMVj4kAAOArcAO4GDix8U0koMRmltXM70zl7gcHt8GLirp/5kLXge2JTkeuB2YKaqzlXVeWAG2HMZP1+StEqDhn4Bv5fkeJIDrXZdVb0J0O6vbfUtwOme58612lL1CyQ5kGQ2yez8/PzgWyJJWtGVA7a7tarOJLkWmEnylWXa9vuju5apX1ioOgQcApienr5ovSTp0g000q+qM+3+LPBZFubk32rTNrT7s635HLCt5+lbgTPL1CVJQ7Ji6Cf5riTfs/gY2A38MXAUWDwCZx/wTHt8FLivHcWzC3i7Tf88B+xOsrntwN3dapKkIRlkeuc64LNZOFTiSuB/VNXvJnkROJJkP/AGcHdr/yxwJ3AKeAe4H6CqziV5GHixtXuoqs6t2ZZIklaUqvGdNp+enq7Z2dlRd0NjoIuHZ47xr6bGXJLjPYfXX8AzciWpQwx9SeoQQ1+SOsTQl6QOMfQlqUMMfUnqEENfkjrE0JeAeirUUx08GUCdY+hLPQx/bXSDXmVT2pAMeHWNoa+x0sXLLUjD5PSO1IfTPNqoDH1pGYa/NhpDX5I6xDl9aUwttX/DSy7rcjjSlwbgNI82CkNfkjrE0JekDjH0JalD3JGrTnJ+Xl3lSF+SOmTg0E9yRZKXknyuLd+Q5IUkJ5P8VpKrWv072/Kptn57z2t8stVfT3L7Wm+MtBKPwlHXrWak/3HgRM/yLwOPVNUO4Dywv9X3A+er6vuBR1o7ktwI3AP8ELAH+LUkV1xe9yVJqzFQ6CfZCvwE8Om2HODDwGdak8PAXe3x3rZMW39ba78XeLqqvlFVfwqcAnauxUZIkgYz6Ej/V4GfA/6uLX8f8LWqerctzwFb2uMtwGmAtv7t1v5b9T7P+ZYkB5LMJpmdn59fxaZIklayYugn+QhwtqqO95b7NK0V1i33nG8Xqg5V1XRVTU9NTa3UPWmo3CegSTfIIZu3Ah9NcifwfuADLIz8NyW5so3mtwJnWvs5YBswl+RK4HuBcz31Rb3PkSQNwYoj/ar6ZFVtrartLOyI/XxV3Qt8AfjJ1mwf8Ex7fLQt09Z/vqqq1e9pR/fcAOwAvrhmWyJJWtHlnJz174Gnk/wi8BLweKs/Dvx6klMsjPDvAaiqV5McAV4D3gUeqKpvXsbPlyStUmqMr9M6PT1ds7Ozo+6Ghmi9vi5xrefhc+/ofm/G+FdWYyLJ8aqa7rfOM3KlS+AOXU0qQ1+SOsTQl6QO8Sqb2tCcgpEuZOhLE8bvztXlMPQ1Eut1lI6k5TmnL10Gj+LRpDH0JalDDH1J6hBDX5I6xB252pCcZ5f6c6QvSR1i6EtShxj6ktQhhr4kdYihL0kdYuhLa8AzczUpPGRTG4rBKy3Pkb4kdYihL60hp3k07lYM/STvT/LFJF9O8mqS/9DqNyR5IcnJJL+V5KpW/862fKqt397zWp9s9deT3L5eGyVJ6m+Qkf43gA9X1Y8ANwF7kuwCfhl4pKp2AOeB/a39fuB8VX0/8EhrR5IbgXuAHwL2AL+W5Iq13BhJ0vJWDP1a8Ddt8X3tVsCHgc+0+mHgrvZ4b1umrb8tSVr96ar6RlX9KXAK2LkmWyFJGshAc/pJrkjyMnAWmAH+BPhaVb3bmswBW9rjLcBpgLb+beD7eut9nqMNKul/kzQaA4V+VX2zqm4CtrIwOv9gv2btvt+vdC1Tv0CSA0lmk8zOz88P0j1J0oBWdfROVX0N+ANgF7ApyeJx/luBM+3xHLANoK3/XuBcb73Pc3p/xqGqmq6q6ampqdV0T+o0/6rSIAY5emcqyab2+O8BPwacAL4A/GRrtg94pj0+2pZp6z9fVdXq97Sje24AdgBfXKsNkSStbJAzcq8HDrcjbb4DOFJVn0vyGvB0kl8EXgIeb+0fB349ySkWRvj3AFTVq0mOAK8B7wIPVNU313Zz1FUeGy8NZsXQr6pXgA/1qX+VPkffVNX/A+5e4rV+Cfil1XdTkrQWPCNXWgeematxZehLUocY+pLUIYa+JHWIoS9JHeKXqGiiubNUWh1H+pLUIYa+JHWIoS+tI4/X17gx9CWpQwx9SeoQQ1+SOsTQl6QOMfQlqUM8OUtrwm9okiaDoS8NweJhm7n3oq+FXndLfSDX8LuiMeD0jiR1iCN9TSRPeJIujSN9SeoQQ1+SOsTQl6QOWTH0k2xL8oUkJ5K8muTjrX51kpkkJ9v95lZPkkeTnErySpKbe15rX2t/Msm+9dssSVI/g4z03wV+pqo+COwCHkhyI/AgcKyqdgDH2jLAHcCOdjsAPAYLHxLAQeAWYCdwcPGDQpI0HCuGflW9WVVfao//GjgBbAH2Aodbs8PAXe3xXuDJWvA8sCnJ9cDtwExVnauq88AMsGdNt0aStKxVzekn2Q58CHgBuK6q3oSFDwbg2tZsC3C652lzrbZU/b0/40CS2SSz8/Pzq+meJGkFA4d+ku8Gfhv4RFV9fbmmfWq1TP3CQtWhqpququmpqalBuydJGsBAoZ/kfSwE/lNV9Tut/FabtqHdn231OWBbz9O3AmeWqUud4TdpadQGOXonwOPAiar6lZ5VR4HFI3D2Ac/01O9rR/HsAt5u0z/PAbuTbG47cHe3miRpSAa5DMOtwE8Bf5Tk5Vb7eeBTwJEk+4E3gLvbumeBO4FTwDvA/QBVdS7Jw8CLrd1DVXVuTbZCkjSQFUO/qv43/efjAW7r076AB5Z4rSeAJ1bTQamXUyPS5fGMXEnqEK+yqVXxy1LWxiivr79ouX9Lr7W/cTnSl6QOMfQlqUMMfUnqEENfkjrE0JekDjH0JalDDH1J6hBDXxohL8CmYTP0JalDDH1J6hAvw6CJ4BSItDYc6UtShzjSl3SRpS7G5oXYJp8jfUnqEENfGgMeuqlhcXpHfXndfGljcqQvSR1i6EtShxj6ktQhK4Z+kieSnE3yxz21q5PMJDnZ7je3epI8muRUkleS3NzznH2t/ckk+9Znc6TJ5g5drbdBRvr/HdjzntqDwLGq2gEca8sAdwA72u0A8BgsfEgAB4FbgJ3AwcUPCmk5hqC0tlYM/ar6Q+Dce8p7gcPt8WHgrp76k7XgeWBTkuuB24GZqjpXVeeBGS7+IJEkrbNLndO/rqreBGj317b6FuB0T7u5VluqfpEkB5LMJpmdn5+/xO5JkvpZ6x25/f4Or2XqFxerDlXVdFVNT01NrWnndLGk/03SxnSpof9Wm7ah3Z9t9TlgW0+7rcCZZeqSJoiDhMl3qaF/FFg8Amcf8ExP/b52FM8u4O02/fMcsDvJ5rYDd3erSerDHdhaLytehiHJbwI/ClyTZI6Fo3A+BRxJsh94A7i7NX8WuBM4BbwD3A9QVeeSPAy82No9VFXv3TksSVpnK4Z+VX1siVW39WlbwANLvM4TwBOr6p0kaU15Rq4kdYihL0kdYuhLY8wdulprXk9fY8mgk9aHI31J6hBH+h3hCTSTbfEvn9w7nt9M7hepTw5H+pLUIYa+JHWIoS9JHWLoS1KHuCNXY8VDNZc37jt0Nf4M/Q3EI3Q0bjyqZ/w4vSNJHWLoSxPIyzPoUhn6ktQhhr40wRzxa7XckTuBNtoOW0Ore9zBOzqO9CWpQwx9aQNwmkeDcnpnjG20aRytv0k/ectpn/Vn6I8Bw13SsAx9eifJniSvJzmV5MFh/3yND6ck1s9Ge2+T/jet3lBH+kmuAP4L8OPAHPBikqNV9dow+zEq/ifVsL03+Cd12mcpTget3rCnd3YCp6rqqwBJngb2AhMZ+ob4pdlII9BJs9E/BBat1e/mRvzwGHbobwFO9yzPAbf0NkhyADjQFv8myevr1JdrgL9ap9eeVEN5T3Lvev+ENbeB/69ccjpu4Pfk21b54TFO78k/XGrFsEO/31t4wWdpVR0CDq17R5LZqppe758zSXxP+vN9uZjvycUm5T0Z9o7cOWBbz/JW4MyQ+yBJnTXs0H8R2JHkhiRXAfcAR4fcB0nqrKFO71TVu0n+LfAccAXwRFW9Osw+9Fj3KaQJ5HvSn+/LxXxPLjYR70lqI+6eliT15bV3JKlDDH1J6hBDH0jys0kqyTWj7suoJfmPSb6S5JUkn02yadR9GhUvGXKxJNuSfCHJiSSvJvn4qPs0LpJckeSlJJ8bdV+W0/nQT7KNhctCvDHqvoyJGeAfV9U/Af4P8MkR92ckei4ZcgdwI/CxJDeOtldj4V3gZ6rqg8Au4AHfl2/5OHBi1J1YSedDH3gE+Dnec5JYV1XV71XVu23xeRbOpeiib10ypKr+Fli8ZEinVdWbVfWl9vivWQi5LaPt1egl2Qr8BPDpUfdlJZ0O/SQfBf6iqr486r6MqZ8G/teoOzEi/S4Z0vlw65VkO/Ah4IXR9mQs/CoLg8e/G3VHVrLhr6ef5PeBf9Bn1S8APw/sHm6PRm+596SqnmltfoGFP+WfGmbfxsiKlwzpsiTfDfw28Imq+vqo+zNKST4CnK2q40l+dNT9WcmGD/2q+rF+9SQ/DNwAfDkLV1XaCnwpyc6q+sshdnHolnpPFiXZB3wEuK26eyKHlwxZQpL3sRD4T1XV74y6P2PgVuCjSe4E3g98IMlvVNW/HnG/+vLkrCbJnwHTVTUuV8kbiSR7gF8B/kVVzY+6P6OS5EoWdmTfBvwFC5cQ+VcjPIN8LGRhhHQYOFdVnxh1f8ZNG+n/bFV9ZNR9WUqn5/TV138GvgeYSfJykv866g6NQtuZvXjJkBPAka4HfnMr8FPAh9v/j5fbCFcTwpG+JHWII31J6hBDX5I6xNCXpA4x9CWpQwx9SeoQQ1+SOsTQl6QO+f9FrIbSI5icJQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "y = v0 + beta_f0*(df['t1']==0)*df['gap1'].apply(lambda x: (x+c)**alpha) + beta_f1*(df['t2']==0)*df['gap2'].apply(lambda x: (x+c)**alpha) + beta_f1*(df['t3']==0)*beta_last_tch +\\\n",
    "beta_g0*(df['t1']==1)*df['gap1'].apply(lambda x: (x+c)**alpha) +  beta_g1*(df['t2']==1)*df['gap2'].apply(lambda x: (x+c)**alpha) + beta_g1*(df['t3']==1) +\\\n",
    "beta_s0*(df['t1']==2)*df['gap1'].apply(lambda x: (x+c)**alpha) + beta_s1*(df['t2']==2)*df['gap2'].apply(lambda x: (x+c)**alpha) + beta_s1*(df['t3']==2)*beta_last_tch +\\\n",
    "beta_gender*df['gender']\n",
    "\n",
    "plt.hist(epsilon, bins=50, color='blue')\n",
    "plt.hist(y, bins=50, color='orange')\n",
    "\n",
    "\n",
    "z = 1/(1+exp(-y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['y'] = y \n",
    "df['z'] = z \n",
    "df['epsilon'] = epsilon\n",
    "\n",
    "# Conversion for only top 10% of users \n",
    "df['conversion'] = df['z']>=np.quantile(df['z'], .9)\n",
    "df['conversion']= df['conversion'].astype(int)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Distribution of P(conversion) and **conversion**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAD4CAYAAAAO9oqkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAATe0lEQVR4nO3dfZCd5Xnf8e8vKPglMZawFpdKcldplDQy047JDijNTOpaqRCQQfwBHTFNUKimmnFJmiZpE1HPVB0IM7hpS8vUxlUj1cLj8lKaBE3AIRqMh7ZjYRZjY15CtQWKNhBrXQnVLWM7cq7+cW65x+Ks9uw5+yJpv5+ZM/s813M/51w3Wva3z8s5m6pCkrS0/cBiNyBJWnyGgSTJMJAkGQaSJAwDSRKwbLEbGNTKlStrdHR0sduQpLPK008//Y2qGjm1ftaGwejoKOPj44vdhiSdVZL8z151TxNJkgwDSZJhIEnCMJAkYRhIkjAMJEn0EQZJ9iY5kuS5Htv+UZJKsrKtJ8ldSSaSPJvk0q6x25Icao9tXfWfTPK1ts9dSTJXk5Mk9aefI4NPA5tPLSZZA/wt4LWu8pXAuvbYAdzdxl4I7AIuBy4DdiVZ0fa5u409ud/bXkuSNL9mDIOqegI42mPTncBvAN1/EGELcE91HASWJ7kYuAI4UFVHq+oYcADY3LZdUFVfrM4fVrgHuHa4KUmSZmugdyAnuQb4k6r66ilndVYBh7vWJ1vtdPXJHvXpXncHnaMIPvCBDwzSupaQ0Z0PL8rrvnrH1YvyutIwZn0BOcm7gY8B/7TX5h61GqDeU1XtrqqxqhobGXnbR2tIkgY0yN1EfxlYC3w1yavAauDLSf4Cnd/s13SNXQ28PkN9dY+6JGkBzToMquprVXVRVY1W1SidH+iXVtWfAvuBG9tdRRuA41X1BvAosCnJinbheBPwaNv2zSQb2l1ENwIPzdHcJEl9mvGaQZJ7gQ8DK5NMAruqas80wx8BrgImgLeAmwCq6miS24Cn2rhbq+rkRemP0rlj6V3A59pDOmst1rUK8HqFBjdjGFTVDTNsH+1aLuDmacbtBfb2qI8Dl8zUh85Oi/mDUVL/fAeyJMkwkCQZBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAb84zaSzkz+QR8NyiMDSZJhIEkyDCRJGAaSJAwDSRKGgSQJw0CShGEgScI3nS0Z/mF6Sacz45FBkr1JjiR5rqv220n+OMmzSX4vyfKubbckmUjyUpIruuqbW20iyc6u+tokTyY5lOT+JOfP5QQlSTPr5zTRp4HNp9QOAJdU1V8F/jtwC0CS9cBW4INtn08mOS/JecAngCuB9cANbSzAx4E7q2odcAzYPtSMJEmzNmMYVNUTwNFTan9UVSfa6kFgdVveAtxXVd+uqleACeCy9pioqper6jvAfcCWJAE+AjzY9t8HXDvknCRJszQXF5D/LvC5trwKONy1bbLVpqu/D3izK1hO1ntKsiPJeJLxqampOWhdkgRDhkGSjwEngM+eLPUYVgPUe6qq3VU1VlVjIyMjs21XkjSNge8mSrIN+DlgY1Wd/AE+CazpGrYaeL0t96p/A1ieZFk7OugeL0laIAMdGSTZDPwmcE1VvdW1aT+wNck7kqwF1gFfAp4C1rU7h86nc5F5fwuRx4Hr2v7bgIcGm4okaVD93Fp6L/BF4MeTTCbZDvxb4D3AgSRfSfIpgKp6HngAeAH4Q+Dmqvpu+63/l4BHgReBB9pY6ITKryWZoHMNYc+czlCSNKMZTxNV1Q09ytP+wK6q24Hbe9QfAR7pUX+Zzt1GkqRF4sdRSJIMA0mSYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJ/+ylpDmwWH9W9dU7rl6U1z0XeWQgSTIMJEmGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kSfYRBkr1JjiR5rqt2YZIDSQ61rytaPUnuSjKR5Nkkl3bts62NP5RkW1f9J5N8re1zV5LM9SQlSafXz5HBp4HNp9R2Ao9V1TrgsbYOcCWwrj12AHdDJzyAXcDlwGXArpMB0sbs6Nrv1NeSJM2zGcOgqp4Ajp5S3gLsa8v7gGu76vdUx0FgeZKLgSuAA1V1tKqOAQeAzW3bBVX1xaoq4J6u55IkLZBBrxm8v6reAGhfL2r1VcDhrnGTrXa6+mSPek9JdiQZTzI+NTU1YOuSpFPN9QXkXuf7a4B6T1W1u6rGqmpsZGRkwBYlSacaNAy+3k7x0L4eafVJYE3XuNXA6zPUV/eoS5IW0KBhsB84eUfQNuChrvqN7a6iDcDxdhrpUWBTkhXtwvEm4NG27ZtJNrS7iG7sei5J0gKZ8S+dJbkX+DCwMskknbuC7gAeSLIdeA24vg1/BLgKmADeAm4CqKqjSW4Dnmrjbq2qkxelP0rnjqV3AZ9rD0nSApoxDKrqhmk2bewxtoCbp3mevcDeHvVx4JKZ+pAkzR/fgSxJMgwkSYaBJAnDQJKEYSBJwjCQJGEYSJLo430GmjujOx9e7BYkqSePDCRJhoEkyTCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSQwZBkl+NcnzSZ5Lcm+SdyZZm+TJJIeS3J/k/Db2HW19om0f7XqeW1r9pSRXDDclSdJsDRwGSVYB/wAYq6pLgPOArcDHgTurah1wDNjedtkOHKuqHwXubONIsr7t90FgM/DJJOcN2pckafaGPU20DHhXkmXAu4E3gI8AD7bt+4Br2/KWtk7bvjFJWv2+qvp2Vb0CTACXDdmXJGkWBg6DqvoT4F8Ar9EJgePA08CbVXWiDZsEVrXlVcDhtu+JNv593fUe+3yfJDuSjCcZn5qaGrR1SdIphjlNtILOb/Vrgb8I/BBwZY+hdXKXabZNV397sWp3VY1V1djIyMjsm5Yk9TTMaaKfBV6pqqmq+jPgd4G/Dixvp40AVgOvt+VJYA1A2/5e4Gh3vcc+kqQFMEwYvAZsSPLudu5/I/AC8DhwXRuzDXioLe9v67Ttn6+qavWt7W6jtcA64EtD9CVJmqVlMw/praqeTPIg8GXgBPAMsBt4GLgvyW+12p62yx7gM0km6BwRbG3P83ySB+gEyQng5qr67qB9SVo6Rnc+vGiv/eodVy/aa8+HgcMAoKp2AbtOKb9Mj7uBqupbwPXTPM/twO3D9CJJGpzvQJYkGQaSJMNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgSWLIMEiyPMmDSf44yYtJfirJhUkOJDnUvq5oY5PkriQTSZ5NcmnX82xr4w8l2TbspCRJszPskcG/Af6wqv4K8NeAF4GdwGNVtQ54rK0DXAmsa48dwN0ASS4EdgGXA5cBu04GiCRpYQwcBkkuAH4G2ANQVd+pqjeBLcC+NmwfcG1b3gLcUx0HgeVJLgauAA5U1dGqOgYcADYP2pckafaGOTL4EWAK+A9JnknyO0l+CHh/Vb0B0L5e1MavAg537T/ZatPV3ybJjiTjScanpqaGaF2S1G2YMFgGXArcXVUfAv4v//+UUC/pUavT1N9erNpdVWNVNTYyMjLbfiVJ0xgmDCaByap6sq0/SCccvt5O/9C+Hukav6Zr/9XA66epS5IWyMBhUFV/ChxO8uOttBF4AdgPnLwjaBvwUFveD9zY7iraABxvp5EeBTYlWdEuHG9qNUnSAlk25P6/DHw2yfnAy8BNdALmgSTbgdeA69vYR4CrgAngrTaWqjqa5DbgqTbu1qo6OmRfkqRZGCoMquorwFiPTRt7jC3g5mmeZy+wd5heJEmD8x3IkiTDQJJkGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJOYgDJKcl+SZJH/Q1tcmeTLJoST3Jzm/1d/R1ifa9tGu57il1V9KcsWwPUmSZmcujgx+BXixa/3jwJ1VtQ44Bmxv9e3Asar6UeDONo4k64GtwAeBzcAnk5w3B31Jkvo0VBgkWQ1cDfxOWw/wEeDBNmQfcG1b3tLWads3tvFbgPuq6ttV9QowAVw2TF+SpNkZ9sjgXwO/Afx5W38f8GZVnWjrk8CqtrwKOAzQth9v479X77HP90myI8l4kvGpqakhW5cknTRwGCT5OeBIVT3dXe4xtGbYdrp9vr9YtbuqxqpqbGRkZFb9SpKmt2yIfX8auCbJVcA7gQvoHCksT7Ks/fa/Gni9jZ8E1gCTSZYB7wWOdtVP6t5HkrQABj4yqKpbqmp1VY3SuQD8+ar6O8DjwHVt2Dbgoba8v63Ttn++qqrVt7a7jdYC64AvDdqXJGn2hjkymM5vAvcl+S3gGWBPq+8BPpNkgs4RwVaAqno+yQPAC8AJ4Oaq+u489CVJmsachEFVfQH4Qlt+mR53A1XVt4Drp9n/duD2uehFkjR7vgNZkmQYSJIMA0kShoEkCcNAkoRhIEnCMJAkMT9vOjvjje58eLFbkKQzikcGkiTDQJJkGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEliib4DWZKGtVifZPDqHVfPy/N6ZCBJMgwkSUOEQZI1SR5P8mKS55P8SqtfmORAkkPt64pWT5K7kkwkeTbJpV3Pta2NP5Rk2/DTkiTNxjBHBieAX6+qnwA2ADcnWQ/sBB6rqnXAY20d4EpgXXvsAO6GTngAu4DLgcuAXScDRJK0MAYOg6p6o6q+3Ja/CbwIrAK2APvasH3AtW15C3BPdRwElie5GLgCOFBVR6vqGHAA2DxoX5Kk2ZuTawZJRoEPAU8C76+qN6ATGMBFbdgq4HDXbpOtNl291+vsSDKeZHxqamouWpckMQdhkOSHgf8M/MOq+t+nG9qjVqepv71YtbuqxqpqbGRkZPbNSpJ6GioMkvwgnSD4bFX9bit/vZ3+oX090uqTwJqu3VcDr5+mLklaIMPcTRRgD/BiVf2rrk37gZN3BG0DHuqq39juKtoAHG+nkR4FNiVZ0S4cb2o1SdICGeYdyD8N/ALwtSRfabV/AtwBPJBkO/AacH3b9ghwFTABvAXcBFBVR5PcBjzVxt1aVUeH6EuSNEsDh0FV/Vd6n+8H2NhjfAE3T/Nce4G9g/YiSRqO70CWJBkGkiTDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkcQaFQZLNSV5KMpFk52L3I0lLyRkRBknOAz4BXAmsB25Isn5xu5KkpeOMCAPgMmCiql6uqu8A9wFbFrknSVoyli12A80q4HDX+iRw+amDkuwAdrTV/5PkpVm8xkrgGwN3ePZy3kvHUpwzLLF55+PfWxx03n+pV/FMCYP0qNXbClW7gd0DvUAyXlVjg+x7NnPeS8dSnDM477l6vjPlNNEksKZrfTXw+iL1IklLzpkSBk8B65KsTXI+sBXYv8g9SdKScUacJqqqE0l+CXgUOA/YW1XPz/HLDHR66RzgvJeOpThncN5zIlVvOzUvSVpizpTTRJKkRWQYSJLOvTCY6WMtkrwjyf1t+5NJRhe+y7nXx7x/LckLSZ5N8liSnvcan036/QiTJNclqSTnxO2H/cw7yd9u/97PJ/mPC93jfOjje/wDSR5P8kz7Pr9qMfqcS0n2JjmS5LlptifJXe2/ybNJLh34xarqnHnQufj8P4AfAc4HvgqsP2XM3wc+1Za3Avcvdt8LNO+/Cby7LX/0bJ93P3Nu494DPAEcBMYWu+8F+rdeBzwDrGjrFy123ws0793AR9vyeuDVxe57Dub9M8ClwHPTbL8K+Byd92ptAJ4c9LXOtSODfj7WYguwry0/CGxM0utNb2eTGeddVY9X1Vtt9SCd93Kczfr9CJPbgH8OfGshm5tH/cz77wGfqKpjAFV1ZIF7nA/9zLuAC9ryezkH3qtUVU8AR08zZAtwT3UcBJYnuXiQ1zrXwqDXx1qsmm5MVZ0AjgPvW5Du5k8/8+62nc5vE2ezGeec5EPAmqr6g4VsbJ7182/9Y8CPJflvSQ4m2bxg3c2ffub9z4CfTzIJPAL88sK0tqhm+//+tM6I9xnMoX4+1qKvj744y/Q9pyQ/D4wBf2NeO5p/p51zkh8A7gR+caEaWiD9/Fsvo3Oq6MN0jgD/S5JLqurNee5tPvUz7xuAT1fVv0zyU8Bn2rz/fP7bWzRz9vPsXDsy6OdjLb43JskyOoeTpzsMOxv09XEeSX4W+BhwTVV9e4F6my8zzfk9wCXAF5K8Sud86v5z4CJyv9/jD1XVn1XVK8BLdMLhbNbPvLcDDwBU1ReBd9L5MLdz2Zx9lM+5Fgb9fKzFfmBbW74O+Hy1KzFnsRnn3U6Z/Ds6QXAunEM+7Zyr6nhVrayq0aoapXOd5JqqGl+cdudMP9/jv0/nhgGSrKRz2ujlBe1y7vUz79eAjQBJfoJOGEwtaJcLbz9wY7uraANwvKreGOSJzqnTRDXNx1okuRUYr6r9wB46h48TdI4Iti5ex3Ojz3n/NvDDwH9q18tfq6prFq3pIfU553NOn/N+FNiU5AXgu8A/rqr/tXhdD6/Pef868O+T/CqdUyW/eLb/opfkXjqn+1a2ayG7gB8EqKpP0bk2chUwAbwF3DTwa53l/60kSXPgXDtNJEkagGEgSTIMJEmGgSQJw0CShGEgScIwkCQB/w8E4gGogdlz5QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "_=plt.hist(df['z'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAD4CAYAAAAO9oqkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAPdUlEQVR4nO3df6xfdX3H8efL1vobQVuNa7tdjHWzkiziDdaZuM0aKLhQ/oClZI5qmjVh6Jwz2+r2RxeVRPaLjQR1ne0oxlkYM6PRuoYAxm2RykUcWhjhrjC4g8l1LZ0bUay+98f3g/uuvbf3lHvv99vbPh/JN/ec9/mccz6f3tu+7vl8z/c0VYUk6fT2vGF3QJI0fIaBJMkwkCQZBpIkDANJErB42B14rpYuXVojIyPD7oYkLRj33HPPd6pq2VTbFmwYjIyMMDY2NuxuSNKCkeTfptvmNJEkyTCQJBkGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkljAn0CejZEtXxzKeR/5+LuGcl5JmolXBpIkw0CSZBhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiS6BgGST6YZH+SbyX5XJIXJjk7yb4kDyW5KcmS1vYFbX28bR/pO86HW/3BJBf01de12niSLXM9SEnS8c0YBkmWA78BjFbVOcAiYANwDXBtVa0CDgGb2i6bgENV9Trg2taOJKvbfm8E1gGfSLIoySLgeuBCYDVweWsrSRqQrtNEi4EXJVkMvBh4AngHcEvbvhO4pC2vb+u07WuTpNV3VdX3q+phYBw4r73Gq+pAVT0D7GptJUkDMmMYVNW/A38MPEovBA4D9wBPVdWR1mwCWN6WlwOPtX2PtPav7K8ftc909WMk2ZxkLMnY5ORkl/FJkjroMk10Fr3f1M8GfgJ4Cb0pnaPVs7tMs+1E68cWq7ZV1WhVjS5btmymrkuSOuoyTfRO4OGqmqyqHwCfB34OOLNNGwGsAB5vyxPASoC2/eXAwf76UftMV5ckDUiXMHgUWJPkxW3ufy1wP3AncGlrsxG4tS3vbuu07XdUVbX6hna30dnAKuBrwN3AqnZ30hJ6bzLvnv3QJEldLZ6pQVXtS3IL8HXgCHAvsA34IrArycdabXvbZTvwmSTj9K4INrTj7E9yM70gOQJcVVU/BEjyPmAvvTuVdlTV/rkboiRpJjOGAUBVbQW2HlU+QO9OoKPbfg+4bJrjXA1cPUV9D7CnS18kSXPPTyBLkgwDSZJhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJLoGAZJzkxyS5J/SfJAkrcmeUWS25I81L6e1domyXVJxpPcl+TcvuNsbO0fSrKxr/7mJN9s+1yXJHM/VEnSdLpeGfw58PdV9TPAzwIPAFuA26tqFXB7Wwe4EFjVXpuBTwIkeQWwFXgLcB6w9dkAaW029+23bnbDkiSdiBnDIMkZwNuB7QBV9UxVPQWsB3a2ZjuBS9ryeuDG6rkLODPJa4ALgNuq6mBVHQJuA9a1bWdU1VerqoAb+44lSRqALlcGrwUmgb9Kcm+STyd5CfDqqnoCoH19VWu/HHisb/+JVjtefWKK+jGSbE4ylmRscnKyQ9clSV10CYPFwLnAJ6vqTcD/8H9TQlOZar6/nkP92GLVtqoararRZcuWHb/XkqTOuoTBBDBRVfva+i30wuHbbYqH9vXJvvYr+/ZfATw+Q33FFHVJ0oDMGAZV9R/AY0l+upXWAvcDu4Fn7wjaCNzalncDV7S7itYAh9s00l7g/CRntTeOzwf2tm3fTbKm3UV0Rd+xJEkDsLhju/cDn02yBDgAvJdekNycZBPwKHBZa7sHuAgYB55ubamqg0k+Ctzd2n2kqg625SuBG4AXAV9qL0nSgHQKg6r6BjA6xaa1U7Qt4KppjrMD2DFFfQw4p0tfJElzz08gS5IMA0mSYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSOIEwSLIoyb1JvtDWz06yL8lDSW5KsqTVX9DWx9v2kb5jfLjVH0xyQV99XauNJ9kyd8OTJHVxIlcGHwAe6Fu/Bri2qlYBh4BNrb4JOFRVrwOube1IshrYALwRWAd8ogXMIuB64EJgNXB5aytJGpBOYZBkBfAu4NNtPcA7gFtak53AJW15fVunbV/b2q8HdlXV96vqYWAcOK+9xqvqQFU9A+xqbSVJA9L1yuDPgN8BftTWXwk8VVVH2voEsLwtLwceA2jbD7f2P64ftc909WMk2ZxkLMnY5ORkx65LkmYyYxgk+SXgyaq6p788RdOaYduJ1o8tVm2rqtGqGl22bNlxei1JOhGLO7R5G3BxkouAFwJn0LtSODPJ4vbb/wrg8dZ+AlgJTCRZDLwcONhXf1b/PtPVJUkDMOOVQVV9uKpWVNUIvTeA76iqXwHuBC5tzTYCt7bl3W2dtv2OqqpW39DuNjobWAV8DbgbWNXuTlrSzrF7TkYnSeqky5XBdH4X2JXkY8C9wPZW3w58Jsk4vSuCDQBVtT/JzcD9wBHgqqr6IUCS9wF7gUXAjqraP4t+SZJO0AmFQVV9GfhyWz5A706go9t8D7hsmv2vBq6eor4H2HMifZEkzR0/gSxJMgwkSYaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgSaJDGCRZmeTOJA8k2Z/kA63+iiS3JXmofT2r1ZPkuiTjSe5Lcm7fsTa29g8l2dhXf3OSb7Z9rkuS+RisJGlqXa4MjgAfqqo3AGuAq5KsBrYAt1fVKuD2tg5wIbCqvTYDn4ReeABbgbcA5wFbnw2Q1mZz337rZj80SVJXM4ZBVT1RVV9vy98FHgCWA+uBna3ZTuCStrweuLF67gLOTPIa4ALgtqo6WFWHgNuAdW3bGVX11aoq4Ma+Y0mSBuCE3jNIMgK8CdgHvLqqnoBeYACvas2WA4/17TbRaserT0xRn+r8m5OMJRmbnJw8ka5Lko6jcxgkeSnwt8BvVtV/Ha/pFLV6DvVji1Xbqmq0qkaXLVs2U5clSR11CoMkz6cXBJ+tqs+38rfbFA/t65OtPgGs7Nt9BfD4DPUVU9QlSQPS5W6iANuBB6rqT/s27QaevSNoI3BrX/2KdlfRGuBwm0baC5yf5Kz2xvH5wN627btJ1rRzXdF3LEnSACzu0OZtwK8C30zyjVb7PeDjwM1JNgGPApe1bXuAi4Bx4GngvQBVdTDJR4G7W7uPVNXBtnwlcAPwIuBL7SVJGpAZw6Cq/pGp5/UB1k7RvoCrpjnWDmDHFPUx4JyZ+iJJmh9+AlmSZBhIkgwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEl0+28vJUlHGdnyxaGc95GPv2tejuuVgSTJMJAkGQaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkTqIwSLIuyYNJxpNsGXZ/JOl0clKEQZJFwPXAhcBq4PIkq4fbK0k6fZwUYQCcB4xX1YGqegbYBawfcp8k6bSxeNgdaJYDj/WtTwBvObpRks3A5rb630kefI7nWwp85znu+5zlmkGf8f8ZypiH7HQb8+k2XjgNx5xrZjXmn5puw8kSBpmiVscUqrYB22Z9smSsqkZne5yFxDGf+k638YJjnksnyzTRBLCyb30F8PiQ+iJJp52TJQzuBlYlOTvJEmADsHvIfZKk08ZJMU1UVUeSvA/YCywCdlTV/nk85aynmhYgx3zqO93GC455zqTqmKl5SdJp5mSZJpIkDZFhIEk6tcNgpkdcJHlBkpva9n1JRgbfy7nTYby/leT+JPcluT3JtPccLxRdH2OS5NIklWTB34bYZcxJfrl9r/cn+etB93GudfjZ/skkdya5t/18XzSMfs6VJDuSPJnkW9NsT5Lr2p/HfUnOnfVJq+qUfNF7I/pfgdcCS4B/BlYf1ebXgU+15Q3ATcPu9zyP9xeBF7flKxfyeLuOubV7GfAV4C5gdNj9HsD3eRVwL3BWW3/VsPs9gDFvA65sy6uBR4bd71mO+e3AucC3ptl+EfAlep/RWgPsm+05T+Urgy6PuFgP7GzLtwBrk0z1AbiFYMbxVtWdVfV0W72L3uc5FrKujzH5KPCHwPcG2bl50mXMvwZcX1WHAKrqyQH3ca51GXMBZ7Tll7PAP6dUVV8BDh6nyXrgxuq5CzgzyWtmc85TOQymesTF8unaVNUR4DDwyoH0bu51GW+/TfR+s1jIZhxzkjcBK6vqC4Ps2Dzq8n1+PfD6JP+U5K4k6wbWu/nRZcx/ALw7yQSwB3j/YLo2NCf6931GJ8XnDOZJl0dcdHoMxgLReSxJ3g2MAj8/rz2af8cdc5LnAdcC7xlUhwagy/d5Mb2pol+gd/X3D0nOqaqn5rlv86XLmC8HbqiqP0nyVuAzbcw/mv/uDcWc/9t1Kl8ZdHnExY/bJFlM7/LyeJdmJ7NOj/RI8k7g94GLq+r7A+rbfJlpzC8DzgG+nOQRenOruxf4m8hdf65vraofVNXDwIP0wmGh6jLmTcDNAFX1VeCF9B5id6qa80f4nMph0OURF7uBjW35UuCOau/OLEAzjrdNmfwFvSBY6PPIMMOYq+pwVS2tqpGqGqH3PsnFVTU2nO7OiS4/139H72YBkiylN210YKC9nFtdxvwosBYgyRvohcHkQHs5WLuBK9pdRWuAw1X1xGwOeMpOE9U0j7hI8hFgrKp2A9vpXU6O07si2DC8Hs9Ox/H+EfBS4G/a++SPVtXFQ+v0LHUc8yml45j3AucnuR/4IfDbVfWfw+v17HQc84eAv0zyQXrTJe9ZwL/YkeRz9Kb5lrb3QbYCzweoqk/Re1/kImAceBp476zPuYD/vCRJc+RUniaSJHVkGEiSDANJkmEgScIwkCRhGEiSMAwkScD/Aml7ikOLe8PCAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "_=plt.hist(df['conversion'])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Create columns for the calculated contribution of each channel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "df['cont_unscaled_fb'] = beta_f0*(df['t1']==0)*df['gap1'].apply(lambda x: (x+c)**alpha) + beta_f1*(df['t2']==0)*df['gap2'].apply(lambda x: (x+c)**alpha) + beta_f1*(df['t3']==0)*beta_last_tch \n",
    "\n",
    "df['cont_unscaled_gg'] = beta_g0*(df['t1']==1)*df['gap1'].apply(lambda x: (x+c)**alpha) +  beta_g1*(df['t2']==1)*df['gap2'].apply(lambda x: (x+c)**alpha) + beta_g1*(df['t3']==1) \n",
    "    \n",
    "df['cont_unscaled_sp'] = beta_s0*(df['t1']==2)*df['gap1'].apply(lambda x: (x+c)**alpha) + beta_s1*(df['t2']==2)*df['gap2'].apply(lambda x: (x+c)**alpha) + beta_s1*(df['t3']==2)*beta_last_tch \n",
    "\n",
    "\n",
    "df['cont_fb'] = df['cont_unscaled_fb']/(df['cont_unscaled_fb'] + df['cont_unscaled_gg'] + df['cont_unscaled_sp'])\n",
    "df['cont_gg'] = df['cont_unscaled_gg']/(df['cont_unscaled_fb'] + df['cont_unscaled_gg'] + df['cont_unscaled_sp'])\n",
    "df['cont_sp'] = df['cont_unscaled_sp']/(df['cont_unscaled_fb'] + df['cont_unscaled_gg'] + df['cont_unscaled_sp'])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### View of sample of the dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>t1</th>\n",
       "      <th>t2</th>\n",
       "      <th>t3</th>\n",
       "      <th>gap1</th>\n",
       "      <th>gap2</th>\n",
       "      <th>gender</th>\n",
       "      <th>y</th>\n",
       "      <th>z</th>\n",
       "      <th>epsilon</th>\n",
       "      <th>conversion</th>\n",
       "      <th>cont_unscaled_fb</th>\n",
       "      <th>cont_unscaled_gg</th>\n",
       "      <th>cont_unscaled_sp</th>\n",
       "      <th>cont_fb</th>\n",
       "      <th>cont_gg</th>\n",
       "      <th>cont_sp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>46446</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2.991232</td>\n",
       "      <td>1.839293</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.509028</td>\n",
       "      <td>0.375421</td>\n",
       "      <td>-0.546641</td>\n",
       "      <td>0</td>\n",
       "      <td>0.239292</td>\n",
       "      <td>1.909238</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.111375</td>\n",
       "      <td>0.888625</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9347</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3.104490</td>\n",
       "      <td>0.785314</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.229195</td>\n",
       "      <td>0.442951</td>\n",
       "      <td>-0.058407</td>\n",
       "      <td>0</td>\n",
       "      <td>0.232716</td>\n",
       "      <td>2.202217</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.095574</td>\n",
       "      <td>0.904426</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95799</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1.721027</td>\n",
       "      <td>1.081446</td>\n",
       "      <td>0</td>\n",
       "      <td>0.632857</td>\n",
       "      <td>0.653137</td>\n",
       "      <td>0.754827</td>\n",
       "      <td>0</td>\n",
       "      <td>1.804070</td>\n",
       "      <td>0.679683</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.726349</td>\n",
       "      <td>0.273651</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89721</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.911207</td>\n",
       "      <td>0.675142</td>\n",
       "      <td>0</td>\n",
       "      <td>1.274928</td>\n",
       "      <td>0.781585</td>\n",
       "      <td>1.129059</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.254071</td>\n",
       "      <td>0.497520</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.819188</td>\n",
       "      <td>0.180812</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23656</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2.952720</td>\n",
       "      <td>2.472946</td>\n",
       "      <td>1</td>\n",
       "      <td>-2.145480</td>\n",
       "      <td>0.104754</td>\n",
       "      <td>-1.903746</td>\n",
       "      <td>0</td>\n",
       "      <td>0.435297</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.433887</td>\n",
       "      <td>0.232881</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.767119</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44667</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1.987761</td>\n",
       "      <td>1.745051</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.231895</td>\n",
       "      <td>0.442285</td>\n",
       "      <td>0.149550</td>\n",
       "      <td>0</td>\n",
       "      <td>1.772913</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.451364</td>\n",
       "      <td>0.797074</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.202926</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98822</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1.919647</td>\n",
       "      <td>1.667138</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.408461</td>\n",
       "      <td>0.399281</td>\n",
       "      <td>0.022298</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.710471</td>\n",
       "      <td>0.464493</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.786437</td>\n",
       "      <td>0.213563</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84398</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>3.143899</td>\n",
       "      <td>3.121115</td>\n",
       "      <td>1</td>\n",
       "      <td>-1.844278</td>\n",
       "      <td>0.136546</td>\n",
       "      <td>-1.675065</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.410313</td>\n",
       "      <td>0.531390</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.726328</td>\n",
       "      <td>0.273672</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29884</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.784579</td>\n",
       "      <td>1.669253</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.294520</td>\n",
       "      <td>0.426898</td>\n",
       "      <td>-0.050631</td>\n",
       "      <td>0</td>\n",
       "      <td>2.361833</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30239</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2.038157</td>\n",
       "      <td>0.274497</td>\n",
       "      <td>1</td>\n",
       "      <td>1.654016</td>\n",
       "      <td>0.839433</td>\n",
       "      <td>1.077247</td>\n",
       "      <td>1</td>\n",
       "      <td>1.495104</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.192582</td>\n",
       "      <td>0.556279</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.443721</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       t1  t2  t3      gap1      gap2  gender         y         z   epsilon  \\\n",
       "46446  0   1   1   2.991232  1.839293  1      -0.509028  0.375421 -0.546641   \n",
       "9347   0   1   1   3.104490  0.785314  0      -0.229195  0.442951 -0.058407   \n",
       "95799  0   1   0   1.721027  1.081446  0       0.632857  0.653137  0.754827   \n",
       "89721  2   1   1   0.911207  0.675142  0       1.274928  0.781585  1.129059   \n",
       "23656  2   0   2   2.952720  2.472946  1      -2.145480  0.104754 -1.903746   \n",
       "44667  0   2   0   1.987761  1.745051  0      -0.231895  0.442285  0.149550   \n",
       "98822  1   2   1   1.919647  1.667138  0      -0.408461  0.399281  0.022298   \n",
       "84398  2   2   1   3.143899  3.121115  1      -1.844278  0.136546 -1.675065   \n",
       "29884  0   0   0   1.784579  1.669253  0      -0.294520  0.426898 -0.050631   \n",
       "30239  0   0   2   2.038157  0.274497  1       1.654016  0.839433  1.077247   \n",
       "\n",
       "       conversion  cont_unscaled_fb  cont_unscaled_gg  cont_unscaled_sp  \\\n",
       "46446  0           0.239292          1.909238          0.000000           \n",
       "9347   0           0.232716          2.202217          0.000000           \n",
       "95799  0           1.804070          0.679683          0.000000           \n",
       "89721  0           0.000000          2.254071          0.497520           \n",
       "23656  0           0.435297          0.000000          1.433887           \n",
       "44667  0           1.772913          0.000000          0.451364           \n",
       "98822  0           0.000000          1.710471          0.464493           \n",
       "84398  0           0.000000          1.410313          0.531390           \n",
       "29884  0           2.361833          0.000000          0.000000           \n",
       "30239  1           1.495104          0.000000          1.192582           \n",
       "\n",
       "        cont_fb   cont_gg   cont_sp  \n",
       "46446  0.111375  0.888625  0.000000  \n",
       "9347   0.095574  0.904426  0.000000  \n",
       "95799  0.726349  0.273651  0.000000  \n",
       "89721  0.000000  0.819188  0.180812  \n",
       "23656  0.232881  0.000000  0.767119  \n",
       "44667  0.797074  0.000000  0.202926  \n",
       "98822  0.000000  0.786437  0.213563  \n",
       "84398  0.000000  0.726328  0.273672  \n",
       "29884  1.000000  0.000000  0.000000  \n",
       "30239  0.556279  0.000000  0.443721  "
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[['t1', 't2', 't3', 'gap1',  'gap2', 'gender','y','z','epsilon', 'conversion', 'cont_unscaled_fb','cont_unscaled_gg','cont_unscaled_sp', 'cont_fb', 'cont_gg', 'cont_sp']].sample(10)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Distribution of epsilon for conversion==1\n",
    "###### P(epsilon | conversion=1) has a higher mean than P(epsilon | conversion=0), as expected"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([1.060e+02, 8.580e+02, 2.348e+03, 3.226e+03, 2.216e+03, 9.200e+02,\n",
       "        2.530e+02, 5.800e+01, 1.400e+01, 1.000e+00]),\n",
       " array([-0.09840525,  0.36791775,  0.83424074,  1.30056373,  1.76688672,\n",
       "         2.23320971,  2.6995327 ,  3.16585569,  3.63217868,  4.09850167,\n",
       "         4.56482466]),\n",
       " <a list of 10 Patch objects>)"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD4CAYAAAAAczaOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAP+UlEQVR4nO3df6xfdX3H8efLAmqmGTguhLXNLtm6RVxiMU1twj8OHBRYVkxGUpJpY0jqHyXBxGQp/oM/RoLJlMVESeporJuTNVNDI81YhxhjMoGLVqRUwh12cm1Dryuixoyl+N4f99P4Be6v3t77/VI+z0fyzfec9/mccz7npH19Tz/f8z1NVSFJ6sMbRt0BSdLwGPqS1BFDX5I6YuhLUkcMfUnqyDmj7sB8LrzwwhofHx91NyTprPLYY4/9rKrGZlv2mg798fFxJiYmRt0NSTqrJPnvuZY5vCNJHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR15Tf8iV2eP8Z33j2S/R+68fiT7lc5WXulLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6siCoZ/kTUkeSfKDJIeSfLzVL03ycJKnk/xLkvNa/Y1tfrItHx/Y1m2t/lSSa1bqoCRJs1vMlf6LwJVV9U5gPbA5ySbgU8BdVbUOeB64ubW/GXi+qv4IuKu1I8llwFbgHcBm4PNJVi3nwUiS5rdg6NeMX7XZc9urgCuBf231PcANbXpLm6ctvypJWv3eqnqxqn4MTAIbl+UoJEmLsqgx/SSrkhwEjgMHgP8Cfl5VJ1uTKWB1m14NPAvQlr8A/N5gfZZ1Bve1PclEkonp6enTPyJJ0pwWFfpV9VJVrQfWMHN1/vbZmrX3zLFsrvor97WrqjZU1YaxsbHFdE+StEindfdOVf0c+BawCTg/yann8a8BjrbpKWAtQFv+u8CJwfos60iShmAxd++MJTm/Tb8ZeC9wGHgI+KvWbBtwX5ve1+Zpy79ZVdXqW9vdPZcC64BHlutAJEkLW8z/nHUJsKfdafMGYG9VfSPJk8C9Sf4W+D5wT2t/D/CPSSaZucLfClBVh5LsBZ4ETgI7quql5T0cSdJ8Fgz9qnocuHyW+jPMcvdNVf0vcOMc27oDuOP0uylJWg7+IleSOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0JekjiwY+knWJnkoyeEkh5Lc2uofS/LTJAfb67qBdW5LMpnkqSTXDNQ3t9pkkp0rc0iSpLmcs4g2J4GPVNX3krwVeCzJgbbsrqr6u8HGSS4DtgLvAH4f+I8kf9wWfw74c2AKeDTJvqp6cjkORJK0sAVDv6qOAcfa9C+THAZWz7PKFuDeqnoR+HGSSWBjWzZZVc8AJLm3tTX0JWlITmtMP8k4cDnwcCvdkuTxJLuTXNBqq4FnB1abarW56q/cx/YkE0kmpqenT6d7kqQFLDr0k7wF+Crw4ar6BXA38IfAemb+JfDpU01nWb3mqb+8ULWrqjZU1YaxsbHFdk+StAiLGdMnybnMBP6Xq+prAFX13MDyLwDfaLNTwNqB1dcAR9v0XHVJ0hAs5u6dAPcAh6vqMwP1SwaavQ94ok3vA7YmeWOSS4F1wCPAo8C6JJcmOY+ZL3v3Lc9hSJIWYzFX+lcA7wd+mORgq30UuCnJemaGaI4AHwKoqkNJ9jLzBe1JYEdVvQSQ5BbgAWAVsLuqDi3jsUiSFrCYu3e+w+zj8fvnWecO4I5Z6vvnW0+StLL8Ra4kdcTQl6SOGPqS1BFDX5I6sqj79HX2GN95/6i7IOk1zCt9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkf8cZbOaqP8MdqRO68f2b6lpfJKX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOrJg6CdZm+ShJIeTHEpya6u/LcmBJE+39wtaPUk+m2QyyeNJ3jWwrW2t/dNJtq3cYUmSZrOYK/2TwEeq6u3AJmBHksuAncCDVbUOeLDNA1wLrGuv7cDdMPMhAdwOvBvYCNx+6oNCkjQcC4Z+VR2rqu+16V8Ch4HVwBZgT2u2B7ihTW8BvlQzvgucn+QS4BrgQFWdqKrngQPA5mU9GknSvE5rTD/JOHA58DBwcVUdg5kPBuCi1mw18OzAalOtNlf9lfvYnmQiycT09PTpdE+StIBFh36StwBfBT5cVb+Yr+kstZqn/vJC1a6q2lBVG8bGxhbbPUnSIiwq9JOcy0zgf7mqvtbKz7VhG9r78VafAtYOrL4GODpPXZI0JIu5eyfAPcDhqvrMwKJ9wKk7cLYB9w3UP9Du4tkEvNCGfx4Ark5yQfsC9+pWkyQNyWL+E5UrgPcDP0xysNU+CtwJ7E1yM/AT4Ma2bD9wHTAJ/Br4IEBVnUjySeDR1u4TVXViWY5CkrQoC4Z+VX2H2cfjAa6apX0BO+bY1m5g9+l0UJK0fPxFriR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdWTD0k+xOcjzJEwO1jyX5aZKD7XXdwLLbkkwmeSrJNQP1za02mWTn8h+KJGkhi7nS/yKweZb6XVW1vr32AyS5DNgKvKOt8/kkq5KsAj4HXAtcBtzU2kqShuichRpU1beTjC9ye1uAe6vqReDHSSaBjW3ZZFU9A5Dk3tb2ydPusSRpyc5kTP+WJI+34Z8LWm018OxAm6lWm6suSRqipYb+3cAfAuuBY8CnWz2ztK156q+SZHuSiSQT09PTS+yeJGk2Swr9qnquql6qqt8AX+C3QzhTwNqBpmuAo/PUZ9v2rqraUFUbxsbGltI9SdIclhT6SS4ZmH0fcOrOnn3A1iRvTHIpsA54BHgUWJfk0iTnMfNl776ld1uStBQLfpGb5CvAe4ALk0wBtwPvSbKemSGaI8CHAKrqUJK9zHxBexLYUVUvte3cAjwArAJ2V9WhZT8aSdK8FnP3zk2zlO+Zp/0dwB2z1PcD+0+rd5KkZeUvciWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqyIKhn2R3kuNJnhiovS3JgSRPt/cLWj1JPptkMsnjSd41sM621v7pJNtW5nAkSfNZzJX+F4HNr6jtBB6sqnXAg20e4FpgXXttB+6GmQ8J4Hbg3cBG4PZTHxSSpOFZMPSr6tvAiVeUtwB72vQe4IaB+pdqxneB85NcAlwDHKiqE1X1PHCAV3+QSJJW2FLH9C+uqmMA7f2iVl8NPDvQbqrV5qpLkoZoub/IzSy1mqf+6g0k25NMJJmYnp5e1s5JUu+WGvrPtWEb2vvxVp8C1g60WwMcnaf+KlW1q6o2VNWGsbGxJXZPkjSbpYb+PuDUHTjbgPsG6h9od/FsAl5owz8PAFcnuaB9gXt1q0mShuichRok+QrwHuDCJFPM3IVzJ7A3yc3AT4AbW/P9wHXAJPBr4IMAVXUiySeBR1u7T1TVK78cliStsAVDv6pummPRVbO0LWDHHNvZDew+rd5JkpbVgqEvaXbjO+8fyX6P3Hn9SPar1wcfwyBJHTH0Jakjhr4kdcQx/RUwqrFeSVqIV/qS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSR84o9JMcSfLDJAeTTLTa25IcSPJ0e7+g1ZPks0kmkzye5F3LcQCSpMVbjiv9P6uq9VW1oc3vBB6sqnXAg20e4FpgXXttB+5ehn1Lkk7DSgzvbAH2tOk9wA0D9S/VjO8C5ye5ZAX2L0maw5mGfgH/nuSxJNtb7eKqOgbQ3i9q9dXAswPrTrXayyTZnmQiycT09PQZdk+SNOicM1z/iqo6muQi4ECSH83TNrPU6lWFql3ALoANGza8arkkaenO6Eq/qo629+PA14GNwHOnhm3a+/HWfApYO7D6GuDomexfknR6lhz6SX4nyVtPTQNXA08A+4Btrdk24L42vQ/4QLuLZxPwwqlhIEnScJzJ8M7FwNeTnNrOP1fVvyV5FNib5GbgJ8CNrf1+4DpgEvg18MEz2LckaQmWHPpV9Qzwzlnq/wNcNUu9gB1L3Z8k6cz5i1xJ6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTnTRytLGrLxnfePbN9H7rx+ZPvW8vBKX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktSR1/WPs0b5IxZJei3ySl+SOmLoS1JHXtfDO5KW16iGTH3mz/IZ+pV+ks1JnkoymWTnsPcvST0baugnWQV8DrgWuAy4Kcllw+yDJPVs2MM7G4HJqnoGIMm9wBbgySH3Q9JZxMdJL59hh/5q4NmB+Sng3YMNkmwHtrfZXyV5agX6cSHwsxXY7tnEczDD8+A5gHnOQT415J4sjz+Ya8GwQz+z1OplM1W7gF0r2olkoqo2rOQ+Xus8BzM8D54D6OscDPuL3Clg7cD8GuDokPsgSd0adug/CqxLcmmS84CtwL4h90GSujXU4Z2qOpnkFuABYBWwu6oODbMPzYoOH50lPAczPA+eA+joHKSqFm4lSXpd8DEMktQRQ1+SOtJd6Pf+GIgku5McT/LEqPsyKknWJnkoyeEkh5LcOuo+jUKSNyV5JMkP2nn4+Kj7NApJViX5fpJvjLovw9BV6PsYCAC+CGwedSdG7CTwkap6O7AJ2NHhnwOAF4Erq+qdwHpgc5JNI+7TKNwKHB51J4alq9Bn4DEQVfV/wKnHQHSjqr4NnBh1P0apqo5V1ffa9C+Z+Qu/erS9Gr6a8as2e257dXVnR5I1wPXAP4y6L8PSW+jP9hiI7v6y67eSjAOXAw+Ptiej0YY2DgLHgQNV1dt5+Hvgb4DfjLojw9Jb6C/4GAj1I8lbgK8CH66qX4y6P6NQVS9V1Xpmfh2/McmfjrpPw5LkL4DjVfXYqPsyTL2Fvo+BEABJzmUm8L9cVV8bdX9Grap+DnyLvr7vuQL4yyRHmBnqvTLJP422Syuvt9D3MRAiSYB7gMNV9ZlR92dUkowlOb9Nvxl4L/Cj0fZqeKrqtqpaU1XjzGTBN6vqr0fcrRXXVehX1Ung1GMgDgN7R/QYiJFJ8hXgP4E/STKV5OZR92kErgDez8yV3cH2um7UnRqBS4CHkjzOzAXRgarq4rbFnvkYBknqSFdX+pLUO0Nfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdeT/Aat8KjAzSnScAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(df[df['conversion']==1]['epsilon'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Distribution of epsilon for conversion==0\n",
    "###### P(epsilon | conversion=0) has a lower mean than P(epsilon | conversion=1), as expected"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([1.4000e+01, 1.7600e+02, 1.0910e+03, 4.5170e+03, 1.2396e+04,\n",
       "        2.1688e+04, 2.5468e+04, 1.8212e+04, 5.8760e+03, 5.6200e+02]),\n",
       " array([-4.21087428, -3.55171342, -2.89255256, -2.2333917 , -1.57423084,\n",
       "        -0.91506997, -0.25590911,  0.40325175,  1.06241261,  1.72157347,\n",
       "         2.38073433]),\n",
       " <a list of 10 Patch objects>)"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAD4CAYAAAAO9oqkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAQB0lEQVR4nO3df6zddX3H8edrrRjjj1CkIGubXbI0i9W5qg104Y+pbKXAYnGRBJZA48hqTMk0cZkFktWIJDVGjWTKUkdDSdBK/BEaW1c7QkJMBHtBxg+ra4MdXNvBZUVkIdEU3/vjfO520p5777n3tufc0ucjOTnf7/t8vt/zPg3h1e/n+6OpKiRJZ7bfG3YDkqThMwwkSYaBJMkwkCRhGEiSgIXDbmC2zj333BoZGRl2G5J0WnnkkUdeqKrFx9dP2zAYGRlhdHR02G1I0mklyX/2qjtNJEkyDCRJhoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIk+rgDOcky4G7gbcDvgK1V9eUknwb+FhhvQ2+uqt1tm5uAG4BXgb+rqj2tvhb4MrAA+Jeq2tLqFwI7gHOAR4Hrquq3J+tHSmeKkU27hvK9h7ZcOZTv1cnTz5HBMeCTVfV2YDWwMcmK9tmXqmple00EwQrgGuAdwFrgq0kWJFkAfAW4HFgBXNu1n8+1fS0HXqQTJJKkAZk2DKrqSFU92pZfBvYDS6bYZB2wo6p+U1W/AA4CF7XXwap6uv2tfwewLkmADwDfattvB66a7Q+SJM3cjM4ZJBkB3g083Eo3Jnk8ybYki1ptCfBs12ZjrTZZ/a3Ar6rq2HF1SdKA9B0GSd4EfBv4RFX9GrgD+ENgJXAE+MLE0B6b1yzqvXrYkGQ0yej4+HivIZKkWegrDJK8jk4Q3FNV3wGoqueq6tWq+h3wNTrTQND5m/2yrs2XAoenqL8AnJ1k4XH1E1TV1qpaVVWrFi8+4XHckqRZmjYM2pz+ncD+qvpiV/2CrmEfAp5syzuBa5K8vl0ltBz4MbAPWJ7kwiRn0TnJvLOqCngA+HDbfj1w39x+liRpJvr5x20uAa4DnkjyWKvdTOdqoJV0pnQOAR8FqKqnktwL/JTOlUgbq+pVgCQ3AnvoXFq6raqeavv7FLAjyWeBn9AJH0nSgEwbBlX1Q3rP6++eYpvbgNt61Hf32q6qnub/p5kkSQPmHciSJMNAkmQYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkYOGwG5Bea0Y27Rp2C9KMeWQgSTIMJEmGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kSfYRBkmVJHkiyP8lTST7e6uck2ZvkQHtf1OpJcnuSg0keT/Kern2tb+MPJFnfVX9vkifaNrcnyan4sZKk3vo5MjgGfLKq3g6sBjYmWQFsAu6vquXA/W0d4HJgeXttAO6ATngAm4GLgYuAzRMB0sZs6Npu7dx/miSpX9OGQVUdqapH2/LLwH5gCbAO2N6GbQeuasvrgLur4yHg7CQXAJcBe6vqaFW9COwF1rbP3lJVP6qqAu7u2pckaQBmdM4gyQjwbuBh4PyqOgKdwADOa8OWAM92bTbWalPVx3rUe33/hiSjSUbHx8dn0rokaQp9h0GSNwHfBj5RVb+eamiPWs2ifmKxamtVraqqVYsXL56uZUlSn/oKgySvoxME91TVd1r5uTbFQ3t/vtXHgGVdmy8FDk9TX9qjLkkakH6uJgpwJ7C/qr7Y9dFOYOKKoPXAfV3169tVRauBl9o00h5gTZJF7cTxGmBP++zlJKvbd13ftS9J0gD08y+dXQJcBzyR5LFWuxnYAtyb5AbgGeDq9tlu4ArgIPAK8BGAqjqa5FZgXxv3mao62pY/BtwFvAH4fntJkgZk2jCoqh/Se14f4NIe4wvYOMm+tgHbetRHgXdO14sk6dTwDmRJkmEgSTIMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJNHf4ygkaUojm3YN5XsPbblyKN/7WuSRgSTJMJAkGQaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJJEH2GQZFuS55M82VX7dJJfJnmsva7o+uymJAeT/DzJZV31ta12MMmmrvqFSR5OciDJN5OcdTJ/oCRpev0cGdwFrO1R/1JVrWyv3QBJVgDXAO9o23w1yYIkC4CvAJcDK4Br21iAz7V9LQdeBG6Yyw+SJM3ctGFQVQ8CR/vc3zpgR1X9pqp+ARwELmqvg1X1dFX9FtgBrEsS4APAt9r224GrZvgbJElzNJdzBjcmebxNIy1qtSXAs11jxlptsvpbgV9V1bHj6j0l2ZBkNMno+Pj4HFqXJHWbbRjcAfwhsBI4Anyh1dNjbM2i3lNVba2qVVW1avHixTPrWJI0qYWz2aiqnptYTvI14HttdQxY1jV0KXC4LfeqvwCcnWRhOzroHi9JGpBZHRkkuaBr9UPAxJVGO4Frkrw+yYXAcuDHwD5gebty6Cw6J5l3VlUBDwAfbtuvB+6bTU+SpNmb9sggyTeA9wHnJhkDNgPvS7KSzpTOIeCjAFX1VJJ7gZ8Cx4CNVfVq28+NwB5gAbCtqp5qX/EpYEeSzwI/Ae48ab9OktSXacOgqq7tUZ70f9hVdRtwW4/6bmB3j/rTdK42kiQNiXcgS5IMA0mSYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiRm+c9eSqeDkU27ht2CdNrwyECSZBhIkgwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAk0UcYJNmW5PkkT3bVzkmyN8mB9r6o1ZPk9iQHkzye5D1d26xv4w8kWd9Vf2+SJ9o2tyfJyf6RkqSp9XNkcBew9rjaJuD+qloO3N/WAS4HlrfXBuAO6IQHsBm4GLgI2DwRIG3Mhq7tjv8uSdIpNm0YVNWDwNHjyuuA7W15O3BVV/3u6ngIODvJBcBlwN6qOlpVLwJ7gbXts7dU1Y+qqoC7u/YlSRqQ2Z4zOL+qjgC09/NafQnwbNe4sVabqj7Wo95Tkg1JRpOMjo+Pz7J1SdLxTvYJ5F7z/TWLek9VtbWqVlXVqsWLF8+yRUnS8WYbBs+1KR7a+/OtPgYs6xq3FDg8TX1pj7okaYBmGwY7gYkrgtYD93XVr29XFa0GXmrTSHuANUkWtRPHa4A97bOXk6xuVxFd37UvSdKALJxuQJJvAO8Dzk0yRueqoC3AvUluAJ4Brm7DdwNXAAeBV4CPAFTV0SS3AvvauM9U1cRJ6Y/RuWLpDcD320uSNEDThkFVXTvJR5f2GFvAxkn2sw3Y1qM+Crxzuj4kSaeOdyBLkgwDSZJhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkAQuH3YAkzdbIpl1D++5DW64c2nefCh4ZSJIMA0mSYSBJYo5hkORQkieSPJZktNXOSbI3yYH2vqjVk+T2JAeTPJ7kPV37Wd/GH0iyfm4/SZI0UyfjyOD9VbWyqla19U3A/VW1HLi/rQNcDixvrw3AHdAJD2AzcDFwEbB5IkAkSYNxKqaJ1gHb2/J24Kqu+t3V8RBwdpILgMuAvVV1tKpeBPYCa09BX5KkScw1DAr4QZJHkmxotfOr6ghAez+v1ZcAz3ZtO9Zqk9VPkGRDktEko+Pj43NsXZI0Ya73GVxSVYeTnAfsTfKzKcamR62mqJ9YrNoKbAVYtWpVzzGSpJmb05FBVR1u788D36Uz5/9cm/6hvT/fho8By7o2XwocnqIuSRqQWYdBkjcmefPEMrAGeBLYCUxcEbQeuK8t7wSub1cVrQZeatNIe4A1SRa1E8drWk2SNCBzmSY6H/hukon9fL2q/jXJPuDeJDcAzwBXt/G7gSuAg8ArwEcAqupokluBfW3cZ6rq6Bz60jwyzMcFSOrfrMOgqp4G/qRH/b+BS3vUC9g4yb62Adtm24skaW68A1mSZBhIkgwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJGDhsBvQYIxs2jXsFiTNYx4ZSJIMA0mSYSBJwnMGkjQrwzoPd2jLladkvx4ZSJLmTxgkWZvk50kOJtk07H4k6UwyL6aJkiwAvgL8BTAG7Euys6p+OtzOTi4v75Q0X82XI4OLgINV9XRV/RbYAawbck+SdMaYF0cGwBLg2a71MeDi4wcl2QBsaKv/k+Tnp7ivc4EXTvF3nAr2PVj2PXina+9z7jufm3MPf9CrOF/CID1qdUKhaiuw9dS305FktKpWDer7Thb7Hiz7HrzTtff53Pd8mSYaA5Z1rS8FDg+pF0k648yXMNgHLE9yYZKzgGuAnUPuSZLOGPNimqiqjiW5EdgDLAC2VdVTQ24LBjgldZLZ92DZ9+Cdrr3P275TdcLUvCTpDDNfpokkSUNkGEiSDIN+JPn7JJXk3GH30q8ktyZ5PMljSX6Q5PeH3VM/knw+yc9a799Ncvawe+pHkquTPJXkd0nm5aWD3U7Hx78k2Zbk+SRPDruXmUiyLMkDSfa3/0Y+PuyeejEMppFkGZ3HZDwz7F5m6PNV9a6qWgl8D/jHYTfUp73AO6vqXcB/ADcNuZ9+PQn8FfDgsBuZTtfjXy4HVgDXJlkx3K76chewdthNzMIx4JNV9XZgNbBxPv55GwbT+xLwD/S4CW4+q6pfd62+kdOk/6r6QVUda6sP0bnnZN6rqv1VdarviD9ZTsvHv1TVg8DRYfcxU1V1pKoebcsvA/vpPHVhXpkXl5bOV0k+CPyyqv496XWT9PyW5DbgeuAl4P1Dbmc2/gb45rCbeA3q6/EvOvmSjADvBh4ebicnOuPDIMm/AW/r8dEtwM3AmsF21L+peq+q+6rqFuCWJDcBNwKbB9rgJKbru425hc7h9T2D7G0q/fR9mujr8S86uZK8Cfg28InjjtznhTM+DKrqz3vVk/wxcCEwcVSwFHg0yUVV9V8DbHFSk/Xew9eBXcyTMJiu7yTrgb8ELq15dCPMDP685zsf/zJgSV5HJwjuqarvDLufXs74MJhMVT0BnDexnuQQsKqqTosnJSZZXlUH2uoHgZ8Ns59+JVkLfAr4s6p6Zdj9vEb93+NfgF/SefzLXw+3pdeudP42eSewv6q+OOx+JuMJ5NeuLUmeTPI4namueXk5Ww//BLwZ2Nsui/3nYTfUjyQfSjIG/CmwK8meYfc0mXaCfuLxL/uBe+fJ41+mlOQbwI+AP0oyluSGYffUp0uA64APtP+mH0tyxbCbOp6Po5AkeWQgSTIMJEkYBpIkDANJEoaBJAnDQJKEYSBJAv4Xt2Oj8HUMN60AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(df[df['conversion']==0]['epsilon'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Format Data for Input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_time_steps = 3\n",
    "num_cats = 3\n",
    "num_feats = 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train Test Split / Split into Numerical and Categorical Matrices / Reshape "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(70000, 3, 1)\n",
      "(70000, 3, 2)\n",
      "(30000, 3, 1)\n",
      "(30000, 3, 2)\n",
      "(70000,)\n",
      "(30000,)\n"
     ]
    }
   ],
   "source": [
    "df['gap3'] = 0 #create an extra gap column for time step 2 which is always 0 since there is no time step 3 in this simulation\n",
    "df['gender_t2'] = df['gender'] #create an extra gender column for time step 2 which is the same as gender\n",
    "df['gender_t3'] = df['gender'] #create an extra gender column for time step 2 which is the same as gender\n",
    "df['log_gap1'] = log(df['gap1'])\n",
    "df['log_gap2'] = log(df['gap2'])\n",
    "df['log_gap3'] = 0\n",
    "\n",
    "X = df[['t1', 't2','t3','gender', 'gap1','log_gap1', 'gender_t2', 'gap2','log_gap2', 'gender_t3', 'gap3','log_gap3', 'epsilon']]\n",
    "y = np.array(df['conversion'])\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.3, random_state=123)\n",
    "\n",
    "#Split X_train into numerical and categorical matrices for separate input into the model\n",
    "#X_train_num = np.array(X_train[['gender', 'gap1','log_gap1', 'gender_t2','gap2', 'log_gap2', 'gender_t3','gap3', 'log_gap3']]) #gender is time-invariant so the same for both time steps\n",
    "#X_test_num = np.array(X_test[['gender', 'gap1','log_gap1', 'gender_t2','gap2', 'log_gap2', 'gender_t3','gap3', 'log_gap3']])\n",
    "X_train_num = np.array(X_train[['gender','log_gap1', 'gender_t2', 'log_gap2', 'gender_t3', 'log_gap3']]) #gender is time-invariant so the same for both time steps\n",
    "X_test_num = np.array(X_test[['gender','log_gap1', 'gender_t2', 'log_gap2', 'gender_t3', 'log_gap3']])\n",
    "\n",
    "X_train_num2 = np.reshape(X_train_num, (len(X_train_num),num_time_steps, num_feats))\n",
    "X_test_num2 = np.reshape(X_test_num, (len(X_test_num),num_time_steps, num_feats))\n",
    "X_train_cat = X_train[['t1', 't2', 't3']]\n",
    "X_train_cat2 = np.reshape(np.array(X_train_cat), (len(X_train_cat),num_time_steps , 1))\n",
    "X_test_cat = X_test[['t1', 't2', 't3']]\n",
    "X_test_cat2 = np.reshape(np.array(X_test_cat), (len(X_test_cat), num_time_steps , 1))\n",
    "print(X_train_cat2.shape)\n",
    "print(X_train_num2.shape)\n",
    "print(X_test_cat2.shape)\n",
    "print(X_test_num2.shape)\n",
    "print(y_train.shape)\n",
    "print(y_test.shape)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### View sample of X_train_num2 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[[ 1.        ,  1.8372666 ],\n",
       "        [ 1.        ,  1.14110961],\n",
       "        [ 1.        ,  0.        ]],\n",
       "\n",
       "       [[ 0.        , -0.40026814],\n",
       "        [ 0.        , -0.43297568],\n",
       "        [ 0.        ,  0.        ]],\n",
       "\n",
       "       [[ 0.        , -0.42064621],\n",
       "        [ 0.        , -1.20213539],\n",
       "        [ 0.        ,  0.        ]]])"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_num2[:3]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Verify X_train_cat2 is tokenized"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>t1</th>\n",
       "      <th>t2</th>\n",
       "      <th>t3</th>\n",
       "      <th>gap1</th>\n",
       "      <th>gap2</th>\n",
       "      <th>gender</th>\n",
       "      <th>y</th>\n",
       "      <th>z</th>\n",
       "      <th>epsilon</th>\n",
       "      <th>conversion</th>\n",
       "      <th>...</th>\n",
       "      <th>cont_unscaled_sp</th>\n",
       "      <th>cont_fb</th>\n",
       "      <th>cont_gg</th>\n",
       "      <th>cont_sp</th>\n",
       "      <th>gap3</th>\n",
       "      <th>gender_t2</th>\n",
       "      <th>gender_t3</th>\n",
       "      <th>log_gap1</th>\n",
       "      <th>log_gap2</th>\n",
       "      <th>log_gap3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2.190819</td>\n",
       "      <td>1.523924</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.950047</td>\n",
       "      <td>0.278875</td>\n",
       "      <td>-0.587738</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.490734</td>\n",
       "      <td>0.781256</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.218744</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.784275</td>\n",
       "      <td>0.421289</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.637772</td>\n",
       "      <td>1.363923</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.626038</td>\n",
       "      <td>0.348409</td>\n",
       "      <td>-0.473866</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.493337</td>\n",
       "      <td>0.310365</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>3.548731</td>\n",
       "      <td>3.432864</td>\n",
       "      <td>0</td>\n",
       "      <td>-1.204429</td>\n",
       "      <td>0.230688</td>\n",
       "      <td>-0.482122</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.280078</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.851292</td>\n",
       "      <td>0.148708</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.266590</td>\n",
       "      <td>1.233395</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2.072739</td>\n",
       "      <td>1.611280</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.590479</td>\n",
       "      <td>0.356525</td>\n",
       "      <td>-0.326713</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.310078</td>\n",
       "      <td>0.867599</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.132401</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.728871</td>\n",
       "      <td>0.477029</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>4.812273</td>\n",
       "      <td>0.483884</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.053244</td>\n",
       "      <td>0.486692</td>\n",
       "      <td>0.158147</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.832799</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.652179</td>\n",
       "      <td>0.347821</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.571170</td>\n",
       "      <td>-0.725911</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99995</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0.917585</td>\n",
       "      <td>0.207132</td>\n",
       "      <td>0</td>\n",
       "      <td>0.770318</td>\n",
       "      <td>0.683590</td>\n",
       "      <td>0.664756</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>2.711284</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.086010</td>\n",
       "      <td>-1.574401</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99996</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>8.081462</td>\n",
       "      <td>2.881253</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.684203</td>\n",
       "      <td>0.335324</td>\n",
       "      <td>0.258067</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.297983</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.219706</td>\n",
       "      <td>0.780294</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2.089573</td>\n",
       "      <td>1.058225</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99997</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>6.574965</td>\n",
       "      <td>5.842555</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.245189</td>\n",
       "      <td>0.439008</td>\n",
       "      <td>0.572657</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.883771</td>\n",
       "      <td>0.116229</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.883269</td>\n",
       "      <td>1.765168</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99998</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2.204649</td>\n",
       "      <td>1.621151</td>\n",
       "      <td>1</td>\n",
       "      <td>0.208894</td>\n",
       "      <td>0.552035</td>\n",
       "      <td>0.356875</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.665189</td>\n",
       "      <td>0.151685</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.848315</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.790568</td>\n",
       "      <td>0.483136</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99999</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1.599053</td>\n",
       "      <td>0.501251</td>\n",
       "      <td>1</td>\n",
       "      <td>1.296248</td>\n",
       "      <td>0.785203</td>\n",
       "      <td>0.763983</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.189535</td>\n",
       "      <td>0.549961</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.450039</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.469412</td>\n",
       "      <td>-0.690649</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100000 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       t1  t2  t3      gap1      gap2  gender         y         z   epsilon  \\\n",
       "0      0   2   0   2.190819  1.523924  0      -0.950047  0.278875 -0.587738   \n",
       "1      0   0   0   1.637772  1.363923  0      -0.626038  0.348409 -0.473866   \n",
       "2      1   2   1   3.548731  3.432864  0      -1.204429  0.230688 -0.482122   \n",
       "3      2   0   0   2.072739  1.611280  0      -0.590479  0.356525 -0.326713   \n",
       "4      1   2   1   4.812273  0.483884  0      -0.053244  0.486692  0.158147   \n",
       "...   ..  ..  ..        ...       ... ..            ...       ...       ...   \n",
       "99995  2   2   2   0.917585  0.207132  0       0.770318  0.683590  0.664756   \n",
       "99996  2   1   2   8.081462  2.881253  0      -0.684203  0.335324  0.258067   \n",
       "99997  0   1   0   6.574965  5.842555  0      -0.245189  0.439008  0.572657   \n",
       "99998  0   2   2   2.204649  1.621151  1       0.208894  0.552035  0.356875   \n",
       "99999  2   2   0   1.599053  0.501251  1       1.296248  0.785203  0.763983   \n",
       "\n",
       "       conversion  ...  cont_unscaled_sp   cont_fb   cont_gg   cont_sp  gap3  \\\n",
       "0      0           ...  0.490734          0.781256  0.000000  0.218744  0      \n",
       "1      0           ...  0.000000          1.000000  0.000000  0.000000  0      \n",
       "2      0           ...  0.280078          0.000000  0.851292  0.148708  0      \n",
       "3      0           ...  0.310078          0.867599  0.000000  0.132401  0      \n",
       "4      0           ...  0.832799          0.000000  0.652179  0.347821  0      \n",
       "...   ..           ...       ...               ...       ...       ... ..      \n",
       "99995  0           ...  2.711284          0.000000  0.000000  1.000000  0      \n",
       "99996  0           ...  1.297983          0.000000  0.219706  0.780294  0      \n",
       "99997  0           ...  0.000000          0.883771  0.116229  0.000000  0      \n",
       "99998  0           ...  1.665189          0.151685  0.000000  0.848315  0      \n",
       "99999  0           ...  1.189535          0.549961  0.000000  0.450039  0      \n",
       "\n",
       "       gender_t2  gender_t3  log_gap1  log_gap2  log_gap3  \n",
       "0      0          0          0.784275  0.421289  0         \n",
       "1      0          0          0.493337  0.310365  0         \n",
       "2      0          0          1.266590  1.233395  0         \n",
       "3      0          0          0.728871  0.477029  0         \n",
       "4      0          0          1.571170 -0.725911  0         \n",
       "...   ..         ..               ...       ... ..         \n",
       "99995  0          0         -0.086010 -1.574401  0         \n",
       "99996  0          0          2.089573  1.058225  0         \n",
       "99997  0          0          1.883269  1.765168  0         \n",
       "99998  1          1          0.790568  0.483136  0         \n",
       "99999  1          1          0.469412 -0.690649  0         \n",
       "\n",
       "[100000 rows x 22 columns]"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[[0],\n",
       "        [1],\n",
       "        [2]],\n",
       "\n",
       "       [[1],\n",
       "        [2],\n",
       "        [2]],\n",
       "\n",
       "       [[0],\n",
       "        [2],\n",
       "        [2]]])"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_cat2[:3]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Deep Learning Model Construction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_size = 20 #dimensionality of embedding vector\n",
    "cardinality = num_chnls # cardinality of the categorical feature (channel)\n",
    "lstm_size = 256"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 261,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras import backend as K\n",
    "\n",
    "class Attention(tf.keras.Model):\n",
    "    def __init__(self, units):\n",
    "        super(Attention, self).__init__()\n",
    "        self.w=self.add_weight(shape=(lstm_size, units), initializer=\"normal\")\n",
    "        self.b=self.add_weight(shape=(3, units), initializer=\"zeros\")\n",
    "        \n",
    "        #self.w2=self.add_weight(shape=(units, 1), initializer=\"normal\")\n",
    "        #self.b2=self.add_weight(shape=(2, 1), initializer=\"zeros\")\n",
    "\n",
    "    def call(self, x, return_sequences=True):\n",
    "\n",
    "        e1 = K.relu(K.dot(x,self.w)+self.b)\n",
    "        \n",
    "        #print(e1)\n",
    "        #e2 = K.relu(K.dot(e1,self.w2)+self.b2)\n",
    "        #print(e2)\n",
    "        a = K.softmax(e1, axis=1)\n",
    "        \n",
    "        #a = tf.keras.activations.softmax(e1, axis=1)\n",
    " \n",
    "        output = a*x\n",
    "        return e1, K.sum(output, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 365,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "inp_cat_data = Input(shape=(num_time_steps, 1)) #input for categorical data\n",
    "inp_num_data = Input(shape=(num_time_steps, num_feats)) #input for numerical data\n",
    "\n",
    "emb = Embedding(cardinality, embedding_size, input_length=(num_time_steps))(inp_cat_data) #Embedding layer for categorical data (channels)\n",
    "\n",
    "resh1 = Reshape((num_time_steps, embedding_size))(emb) #reshape to squash the 4D tensor back into 3D, not sure why emb layer does this\n",
    "\n",
    "conc = Concatenate()([resh1, inp_num_data]) #combine numerical data with channel embedding tensor before inputting into LSTM\n",
    "\n",
    "resh2 = Reshape((num_time_steps, embedding_size+num_feats))(conc) #reshape to fit LSTM needs just in case\n",
    "# LSTM with channel embeddings and control features, returns hidden state sequences for the attention layer to use\n",
    "lstm_layer1 = LSTM(lstm_size, dropout=.05, recurrent_dropout=.05,return_sequences=True)(resh2)\n",
    "lstm_layer2 = LSTM(lstm_size, dropout=.05, recurrent_dropout=.05,return_sequences=True)(lstm_layer1)\n",
    "lstm_layer3 = LSTM(lstm_size, dropout=.05, recurrent_dropout=.05,return_sequences=True)(lstm_layer2)\n",
    "lstm_layer4 = LSTM(256, dropout=.05, recurrent_dropout=.05,return_sequences=True)(lstm_layer3)\n",
    "lstm_layer5 = LSTM(lstm_size, dropout=.05, recurrent_dropout=.05,return_sequences=True)(lstm_layer4)\n",
    "lstm_layer6 = LSTM(lstm_size, dropout=.05, recurrent_dropout=.05,return_sequences=True)(lstm_layer5)\n",
    "#lstm_layer7 = LSTM(lstm_size, dropout=.01, recurrent_dropout=.01,return_sequences=True)(lstm_layer6)\n",
    "\n",
    "#lstm_layer = Bidirectional(LSTM(128, dropout=.01, recurrent_dropout=.01,return_sequences=True))(resh2)\n",
    "\n",
    "\n",
    "#Attention layer: multi-layer perceptron with 1 neuron, returns unscaled attention weight vector and attention weighted sum\n",
    "a_unsc, attention_layer = Attention(1)(lstm_layer6)\n",
    "\n",
    "\n",
    "#Output layer dense, binary outcome\n",
    "out = Dense(1, activation='sigmoid')(attention_layer)\n",
    "\n",
    "#output the prediction and the unscaled attention to retrieve after training - not sure if this screws up the training of any layers\n",
    "model = keras.Model(inputs=[inp_cat_data, inp_num_data], outputs=[out, a_unsc] )\n",
    "\n",
    "#output the prediction only - attention to retrieve after training with intermediate layer extraction\n",
    "#model = keras.Model(inputs=[inp_cat_data, inp_num_data], outputs=out )\n",
    "\n",
    "\n",
    "#model = keras.Model(inputs=[inp_cat_data, inp_num_data], outputs=out )\n",
    "\n",
    "model.compile(optimizer=Adam(0.001),\n",
    "              loss=tf.keras.losses.BinaryCrossentropy(from_logits=False),\n",
    "              metrics=['acc'])\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 366,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_9\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_21 (InputLayer)           [(None, 3, 1)]       0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_10 (Embedding)        (None, 3, 1, 20)     60          input_21[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "reshape_20 (Reshape)            (None, 3, 20)        0           embedding_10[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "input_22 (InputLayer)           [(None, 3, 2)]       0                                            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_10 (Concatenate)    (None, 3, 22)        0           reshape_20[0][0]                 \n",
      "                                                                 input_22[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "reshape_21 (Reshape)            (None, 3, 22)        0           concatenate_10[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lstm_42 (LSTM)                  (None, 3, 256)       285696      reshape_21[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lstm_43 (LSTM)                  (None, 3, 256)       525312      lstm_42[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "lstm_44 (LSTM)                  (None, 3, 256)       525312      lstm_43[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "lstm_45 (LSTM)                  (None, 3, 256)       525312      lstm_44[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "lstm_46 (LSTM)                  (None, 3, 256)       525312      lstm_45[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "lstm_47 (LSTM)                  (None, 3, 256)       525312      lstm_46[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "attention_10 (Attention)        ((None, 3, 1), (None 259         lstm_47[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_9 (Dense)                 (None, 1)            257         attention_10[0][1]               \n",
      "==================================================================================================\n",
      "Total params: 2,912,832\n",
      "Trainable params: 2,912,832\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Run Model until loss stops improving"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 367,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/3\n",
      "700/700 [==============================] - 90s 115ms/step - loss: 0.7695 - dense_9_loss: 0.4076 - attention_10_loss: 0.3619 - dense_9_acc: 0.8990 - attention_10_acc: 0.8985 - val_loss: 0.5908 - val_dense_9_loss: 0.2904 - val_attention_10_loss: 0.3005 - val_dense_9_acc: 0.9024 - val_attention_10_acc: 0.9019\n",
      "Epoch 2/3\n",
      "700/700 [==============================] - 81s 115ms/step - loss: 0.6135 - dense_9_loss: 0.3035 - attention_10_loss: 0.3100 - dense_9_acc: 0.8982 - attention_10_acc: 0.8983 - val_loss: 0.6064 - val_dense_9_loss: 0.3001 - val_attention_10_loss: 0.3063 - val_dense_9_acc: 0.9024 - val_attention_10_acc: 0.9023\n",
      "Epoch 3/3\n",
      "700/700 [==============================] - 80s 114ms/step - loss: 0.6051 - dense_9_loss: 0.2998 - attention_10_loss: 0.3053 - dense_9_acc: 0.8994 - attention_10_acc: 0.8993 - val_loss: 0.5841 - val_dense_9_loss: 0.2897 - val_attention_10_loss: 0.2944 - val_dense_9_acc: 0.9023 - val_attention_10_acc: 0.9023\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x1a8010ef10>"
      ]
     },
     "execution_count": 367,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "epochs = 3\n",
    "batch_size = 100\n",
    "model.fit(x=[X_train_cat2, X_train_num2], y=y_train, validation_data=([X_test_cat2, X_test_num2], y_test), epochs=epochs, batch_size=batch_size)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 291,
   "metadata": {},
   "outputs": [],
   "source": [
    "#intermediate_layer_model = keras.Model(inputs=model.input,\n",
    "                                     #  outputs=model.get_layer(\"attention_56\").output[0])\n",
    "\n",
    "#a = intermediate_layer_model([X_test_cat2, X_test_num2])\n",
    "#a = np.array(a)\n",
    "#print(a.mean())\n",
    "#a[:3]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model Evaluation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Extract prediction probabilities and unscaled attention weight vectors (e1) for observations in X_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 368,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds, a = model.predict(x=[X_test_cat2, X_test_num2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 369,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD6CAYAAABNu5eFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAARrElEQVR4nO3df4xlZ13H8ffH1oIg2EIHrLsbp+gGAwRDHduqCUGr2FLdbWKblKgsuGaDFkGLoYsYGyEkRY1VIjYutLJNSAtWSVdawLVACImtnRYotAU7lNodWuhgS1Ebfqx8/WOehdvdOz/vzJ3uPu9XcnPP+Z7nznmezORzz33OuWdSVUiS+vB9G90BSdL4GPqS1BFDX5I6YuhLUkcMfUnqiKEvSR1ZMvSTXJXkoSSfHbLtD5NUkpPbepK8PclMkjuSnDbQdkeSe9pjx9oOQ5K0HMcvo827gb8Brh4sJtkC/BJw/0D5HGBre5wBXAGckeQZwKXAFFDAbUn2VdUji+345JNPrsnJyWUNRJI077bbbvtqVU0M27Zk6FfVx5NMDtl0OfAG4PqB2nbg6pr/xtfNSU5McgrwEmB/VT0MkGQ/cDZwzWL7npycZHp6eqkuSpIGJPnPhbatak4/yTbgS1X16cM2bQIODKzPttpC9WE/e1eS6STTc3Nzq+meJGkBKw79JE8B3gT8ybDNQ2q1SP3IYtWeqpqqqqmJiaGfTiRJq7SaI/0fA04FPp3kPmAzcHuSH2b+CH7LQNvNwAOL1CVJY7Ti0K+qz1TVs6pqsqommQ/006rqy8A+4BXtKp4zgUer6kHgw8BLk5yU5CTgpa0mSRqj5VyyeQ3wb8Bzk8wm2blI8xuBe4EZ4J3A7wK0E7hvAW5tjzcfOqkrSRqfPJFvrTw1NVVevSNJK5PktqqaGrbNb+RKUkcMfUnqiKEvSR1Zzm0YjjmTu28YWr/vsnPH3BNJGi+P9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SepIl/fTX4j32Zd0rPNIX5I6YuhLUkeWDP0kVyV5KMlnB2p/nuRzSe5I8v4kJw5se2OSmSSfT/LLA/WzW20mye61H4okaSnLOdJ/N3D2YbX9wAuq6oXAfwBvBEjyPOBC4PntNX+b5LgkxwHvAM4Bnge8vLWVJI3RkqFfVR8HHj6s9i9VdbCt3gxsbsvbgWur6ptV9UVgBji9PWaq6t6q+hZwbWsrSRqjtZjT/y3gg215E3BgYNtsqy1UP0KSXUmmk0zPzc2tQfckSYeMFPpJ3gQcBN5zqDSkWS1SP7JYtaeqpqpqamJiYpTuSZIOs+rr9JPsAH4FOKuqDgX4LLBloNlm4IG2vFBdkjQmqzrST3I2cAmwraoeG9i0D7gwyZOSnApsBf4duBXYmuTUJCcwf7J332hdlySt1JJH+kmuAV4CnJxkFriU+at1ngTsTwJwc1W9uqruTPI+4C7mp30uqqr/az/nNcCHgeOAq6rqznUYjyRpEUuGflW9fEj5ykXavxV465D6jcCNK+qdJGlN+Y1cSeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSNL/o9cweTuG4bW77vs3DH3RJJG45G+JHXE0JekjiwZ+kmuSvJQks8O1J6RZH+Se9rzSa2eJG9PMpPkjiSnDbxmR2t/T5Id6zMcSdJilnOk/27g7MNqu4GbqmorcFNbBzgH2Noeu4ArYP5NArgUOAM4Hbj00BuFJGl8lgz9qvo48PBh5e3A3ra8FzhvoH51zbsZODHJKcAvA/ur6uGqegTYz5FvJJKkdbbaOf1nV9WDAO35Wa2+CTgw0G621RaqHyHJriTTSabn5uZW2T1J0jBrfSI3Q2q1SP3IYtWeqpqqqqmJiYk17Zwk9W61of+VNm1De36o1WeBLQPtNgMPLFKXJI3RakN/H3DoCpwdwPUD9Ve0q3jOBB5t0z8fBl6a5KR2AvelrSZJGqMlv5Gb5BrgJcDJSWaZvwrnMuB9SXYC9wMXtOY3Ai8DZoDHgFcBVNXDSd4C3NravbmqDj85LElaZ0uGflW9fIFNZw1pW8BFC/ycq4CrVtQ7SdKa8hu5ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdWTJ++lrYZO7bxhav++yc8fcE0laHo/0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkdGCv0kf5DkziSfTXJNkicnOTXJLUnuSfLeJCe0tk9q6zNt++RaDECStHyrDv0km4DXAlNV9QLgOOBC4G3A5VW1FXgE2NleshN4pKp+HLi8tZMkjdGo0zvHAz+Q5HjgKcCDwC8A17Xte4Hz2vL2tk7bflaSjLh/SdIKrDr0q+pLwF8A9zMf9o8CtwFfq6qDrdkssKktbwIOtNcebO2fefjPTbIryXSS6bm5udV2T5I0xCjTOycxf/R+KvAjwFOBc4Y0rUMvWWTb9wpVe6pqqqqmJiYmVts9SdIQo0zv/CLwxaqaq6pvA/8E/CxwYpvuAdgMPNCWZ4EtAG37DwEPj7B/SdIKjRL69wNnJnlKm5s/C7gL+ChwfmuzA7i+Le9r67TtH6mqI470JUnrZ5Q5/VuYPyF7O/CZ9rP2AJcAFyeZYX7O/sr2kiuBZ7b6xcDuEfotSVqFkW6tXFWXApceVr4XOH1I228AF4yyP0nSaPxGriR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOjHQbBg03ufuGofX7Ljt3zD2RpMfzSF+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSR0YK/SQnJrkuyeeS3J3kZ5I8I8n+JPe055Na2yR5e5KZJHckOW1thiBJWq5Rj/T/GvhQVf0E8JPA3cBu4Kaq2grc1NYBzgG2tscu4IoR9y1JWqFVh36SpwMvBq4EqKpvVdXXgO3A3tZsL3BeW94OXF3zbgZOTHLKqnsuSVqxUY70nwPMAX+f5JNJ3pXkqcCzq+pBgPb8rNZ+E3Bg4PWzrfY4SXYlmU4yPTc3N0L3JEmHGyX0jwdOA66oqhcB/8v3pnKGyZBaHVGo2lNVU1U1NTExMUL3JEmHGyX0Z4HZqrqlrV/H/JvAVw5N27Tnhwbabxl4/WbggRH2L0laoVWHflV9GTiQ5LmtdBZwF7AP2NFqO4Dr2/I+4BXtKp4zgUcPTQNJksZj1P+c9XvAe5KcANwLvIr5N5L3JdkJ3A9c0NreCLwMmAEea2274n/UkrTRRgr9qvoUMDVk01lD2hZw0Sj7kySNxm/kSlJHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdWTUWytrDXjLZUnj4pG+JHXE0Jekjhj6ktQRQ1+SOnJMn8hd6ASpJPXKI31J6oihL0kdMfQlqSMjh36S45J8MskH2vqpSW5Jck+S9yY5odWf1NZn2vbJUfctSVqZtTjSfx1w98D624DLq2or8Aiws9V3Ao9U1Y8Dl7d2kqQxGin0k2wGzgXe1dYD/AJwXWuyFzivLW9v67TtZ7X2kqQxGfVI/6+ANwDfaevPBL5WVQfb+iywqS1vAg4AtO2PtvaPk2RXkukk03NzcyN2T5I0aNWhn+RXgIeq6rbB8pCmtYxt3ytU7amqqaqampiYWG33JElDjPLlrJ8DtiV5GfBk4OnMH/mfmOT4djS/GXigtZ8FtgCzSY4Hfgh4eIT9H/O8+6aktbbqI/2qemNVba6qSeBC4CNV9evAR4HzW7MdwPVteV9bp23/SFUdcaQvSVo/63Gd/iXAxUlmmJ+zv7LVrwSe2eoXA7vXYd+SpEWsyb13qupjwMfa8r3A6UPafAO4YC32J0laHb+RK0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SerIMf2P0Y9Vi/3Dd+/LI2kxHulLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kd8d47x5iF7svjPXkkgUf6ktSVVYd+ki1JPprk7iR3Jnldqz8jyf4k97Tnk1o9Sd6eZCbJHUlOW6tBSJKWZ5TpnYPA66vq9iRPA25Lsh94JXBTVV2WZDewG7gEOAfY2h5nAFe0Z42B0z6SYIQj/ap6sKpub8v/DdwNbAK2A3tbs73AeW15O3B1zbsZODHJKavuuSRpxdZkTj/JJPAi4Bbg2VX1IMy/MQDPas02AQcGXjbbaof/rF1JppNMz83NrUX3JEnNyKGf5AeBfwR+v6q+vljTIbU6olC1p6qmqmpqYmJi1O5JkgaMFPpJvp/5wH9PVf1TK3/l0LRNe36o1WeBLQMv3ww8MMr+JUkrs+oTuUkCXAncXVV/ObBpH7ADuKw9Xz9Qf02Sa5k/gfvooWkgbRxP8Ep9GeXqnZ8DfhP4TJJPtdofMR/270uyE7gfuKBtuxF4GTADPAa8aoR9a535ZiAdm1Yd+lX1CYbP0wOcNaR9ARetdn+SpNH5jVxJ6oj33tGKOO0jHd080pekjnikrzXhJwDp6GDoa10t9GawEN8kpPXl9I4kdcTQl6SOGPqS1BHn9PWE4glhaX0Z+joqrPSEMPhGIQ3j9I4kdcQjfR2znCqSjmToqzu+GahnTu9IUkc80pdWyU8MOhoZ+lKzmiuEpKONoS89QflJQuvB0JfWmJ8Y9ERm6EvHuMXehPzU0B9DXzrKrOUnCaeQ+uMlm5LUEY/0JR1hpZ8A/Gc5R4+xh36Ss4G/Bo4D3lVVl427D5JWZ62mlp6IbxK9THWNNfSTHAe8A/glYBa4Ncm+qrprnP2QdGzzCqqFjftI/3RgpqruBUhyLbAdMPQlLWgjQ3yj9r1enzDGHfqbgAMD67PAGYMNkuwCdrXV/0ny+TH17YniZOCrG92JDeC4+9Pr2Jc17rxtpH386EIbxh36GVKrx61U7QH2jKc7TzxJpqtqaqP7MW6Ouz+9jn2jxz3uSzZngS0D65uBB8bcB0nq1rhD/1Zga5JTk5wAXAjsG3MfJKlbY53eqaqDSV4DfJj5Szavqqo7x9mHo0CvU1uOuz+9jn1Dx52qWrqVJOmY4G0YJKkjhr4kdcTQ3wBJzk7y+SQzSXYP2f7iJLcnOZjk/I3o43pZxtgvTnJXkjuS3JRkweuNjybLGPerk3wmyaeSfCLJ8zain+thqbEPtDs/SSU5Ji7jXMbv/JVJ5trv/FNJfnssHasqH2N8MH8C+wvAc4ATgE8DzzuszSTwQuBq4PyN7vOYx/7zwFPa8u8A793ofo9p3E8fWN4GfGij+z2usbd2TwM+DtwMTG10v8f0O38l8Dfj7ptH+uP33VtRVNW3gEO3oviuqrqvqu4AvrMRHVxHyxn7R6vqsbZ6M/Pf5TjaLWfcXx9YfSqHfWnxKLbk2Ju3AH8GfGOcnVtHyx332Bn64zfsVhSbNqgv47bSse8EPriuPRqPZY07yUVJvsB8+L12TH1bb0uOPcmLgC1V9YFxdmydLfdv/dfaVOZ1SbYM2b7mDP3xW/JWFMewZY89yW8AU8Cfr2uPxmNZ466qd1TVjwGXAH+87r0aj0XHnuT7gMuB14+tR+OxnN/5PwOTVfVC4F+BveveKwz9jdDzrSiWNfYkvwi8CdhWVd8cU9/W00p/59cC561rj8ZnqbE/DXgB8LEk9wFnAvuOgZO5S/7Oq+q/Bv6+3wn81Dg6ZuiPX8+3olhy7O2j/t8xH/gPbUAf18Nyxr11YPVc4J4x9m89LTr2qnq0qk6uqsmqmmT+PM62qpremO6umeX8zk8ZWN0G3D2OjvnvEsesFrgVRZI3A9NVtS/JTwPvB04CfjXJn1bV8zew22tiOWNnfjrnB4F/SAJwf1Vt27BOr4Fljvs17RPOt4FHgB0b1+O1s8yxH3OWOe7XJtkGHAQeZv5qnnXnbRgkqSNO70hSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1JH/B9HYGiayTScaAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#preds = pd.DataFrame(model.predict([X_train_cat2, X_train_num2]))\n",
    "preds = pd.DataFrame(preds)\n",
    "preds.rename(columns={0:'pred_prob'}, inplace=True)\n",
    "_=plt.hist(preds['pred_prob'].sample(10000), bins=50)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Use the conversion balance of the training set to find a reasonable prediction threshold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 370,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.22776051419760499"
      ]
     },
     "execution_count": 370,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "thresh = np.quantile(preds['pred_prob'], (1-y_train.mean()))\n",
    "thresh"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Extract attention weights from their awkward data structure and put them in a dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 371,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "t1_attn_unscaled    0.091096\n",
       "t2_attn_unscaled    0.103275\n",
       "t3_attn_unscaled    0.122331\n",
       "dtype: float64"
      ]
     },
     "execution_count": 371,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "a2 = []\n",
    "for i in range(len(a)):\n",
    "    a2.append( [a[i][0][0], a[i][1][0], a[i][2][0]])\n",
    "a2_df = pd.DataFrame(a2, columns=['t1_attn_unscaled', 't2_attn_unscaled', 't3_attn_unscaled'])\n",
    "a2_df.mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Combine original input features, target, actual channel contributions, predictions, and estimated channel attentions in one convenient dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 372,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "preds_df = pd.concat([preds, pd.DataFrame(y_test)], axis=1)\n",
    "preds_df.rename(columns={0:'actual'}, inplace=True)\n",
    "preds_df.reset_index(drop=True,inplace=True)\n",
    "preds_df['pred'] = preds_df['pred_prob']>thresh\n",
    "preds_df2 = pd.concat([pd.DataFrame(X_test).reset_index(), preds_df], axis=1)\n",
    "preds_df2 = pd.concat([preds_df2, a2_df], axis=1)\n",
    "preds_df3 = preds_df2.merge(df[['t1', 't2', 'gender', 'gap1','cont_fb','cont_gg','cont_sp' ]], 'left', on=['t1', 't2', 'gender', 'gap1'])\n",
    "\n",
    "preds_df3['t1_attn'] = preds_df3['t1_attn_unscaled']/(preds_df3['t1_attn_unscaled']+preds_df3['t2_attn_unscaled']+preds_df3['t3_attn_unscaled'])\n",
    "preds_df3['t2_attn'] = preds_df3['t2_attn_unscaled']/(preds_df3['t1_attn_unscaled']+preds_df3['t2_attn_unscaled']+preds_df3['t3_attn_unscaled'])\n",
    "preds_df3['t3_attn'] = preds_df3['t3_attn_unscaled']/(preds_df3['t1_attn_unscaled']+preds_df3['t2_attn_unscaled']+preds_df3['t3_attn_unscaled'])\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Functions for getting the contribution for the timestep to compare directly to attention weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 373,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_t1_cont(row):\n",
    "    t1_cont = 0\n",
    "\n",
    "    if row['t1']==0:\n",
    "        t1_cont = row['cont_fb']\n",
    "    elif row['t1']==1:\n",
    "        t1_cont = row['cont_gg']\n",
    "    else:\n",
    "        t1_cont = row['cont_sp']\n",
    "\n",
    "    return t1_cont\n",
    "\n",
    "def get_t2_cont(row):\n",
    "    t2_cont = 0\n",
    "\n",
    "    if row['t2']==0:\n",
    "        t2_cont = row['cont_fb']\n",
    "    elif row['t2']==1:\n",
    "        t2_cont = row['cont_gg']\n",
    "    else:\n",
    "        t2_cont = row['cont_sp']\n",
    "\n",
    "    return t2_cont\n",
    "\n",
    "def get_t3_cont(row):\n",
    "    t3_cont = 0\n",
    "\n",
    "    if row['t3']==0:\n",
    "        t3_cont = row['cont_fb']\n",
    "    elif row['t3']==1:\n",
    "        t3_cont = row['cont_gg']\n",
    "    else:\n",
    "        t3_cont = row['cont_sp']\n",
    "\n",
    "    return t3_cont"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 374,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds_df3['t1_cont'] = preds_df3.apply(get_t1_cont, axis=1)\n",
    "preds_df3['t2_cont'] = preds_df3.apply(get_t2_cont, axis=1)\n",
    "preds_df3['t3_cont'] = preds_df3.apply(get_t3_cont, axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Preview a sample of estimated attention (t1_attn, t2_atten) *vs.* actual contribution (t1_cont, t2_cont)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### combine attention for time steps that have the same channel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 375,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds_df3.loc[(preds_df3['t1']==preds_df3['t2']) & (preds_df3['t1']!=preds_df3['t3']), 't1_attn'] = preds_df3['t1_attn']+preds_df3['t2_attn']\n",
    "preds_df3.loc[(preds_df3['t1']==preds_df3['t2']) & (preds_df3['t1']!=preds_df3['t3']), 't2_attn'] = preds_df3['t1_attn']\n",
    "\n",
    "preds_df3.loc[(preds_df3['t1']==preds_df3['t3']) & (preds_df3['t1']!=preds_df3['t2']), 't1_attn'] = preds_df3['t1_attn']+preds_df3['t3_attn']\n",
    "preds_df3.loc[(preds_df3['t1']==preds_df3['t3']) & (preds_df3['t1']!=preds_df3['t2']), 't3_attn'] = preds_df3['t1_attn']\n",
    "\n",
    "preds_df3.loc[(preds_df3['t2']==preds_df3['t3']) & (preds_df3['t2']!=preds_df3['t1']), 't2_attn'] = preds_df3['t2_attn']+preds_df3['t3_attn']\n",
    "preds_df3.loc[(preds_df3['t2']==preds_df3['t3']) & (preds_df3['t2']!=preds_df3['t1']), 't3_attn'] = preds_df3['t2_attn']\n",
    "\n",
    "preds_df3.loc[(preds_df3['t1']==preds_df3['t2']) & (preds_df3['t2']==preds_df3['t3']), 't1_attn'] = preds_df3['t1_attn']+preds_df3['t2_attn']+preds_df3['t3_attn']\n",
    "preds_df3.loc[(preds_df3['t1']==preds_df3['t2']) & (preds_df3['t2']==preds_df3['t3']), 't2_attn'] = preds_df3['t1_attn']\n",
    "preds_df3.loc[(preds_df3['t1']==preds_df3['t2']) & (preds_df3['t2']==preds_df3['t3']), 't23attn'] = preds_df3['t1_attn']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 376,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>t1</th>\n",
       "      <th>t2</th>\n",
       "      <th>t3</th>\n",
       "      <th>t1_cont</th>\n",
       "      <th>t1_attn</th>\n",
       "      <th>t2_cont</th>\n",
       "      <th>t2_attn</th>\n",
       "      <th>t3_cont</th>\n",
       "      <th>t3_attn</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3764</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0.383158</td>\n",
       "      <td>0.659369</td>\n",
       "      <td>0.383158</td>\n",
       "      <td>0.659369</td>\n",
       "      <td>0.616842</td>\n",
       "      <td>0.340631</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27181</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.527001</td>\n",
       "      <td>0.622294</td>\n",
       "      <td>0.527001</td>\n",
       "      <td>0.622294</td>\n",
       "      <td>0.472999</td>\n",
       "      <td>0.377706</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14482</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.115770</td>\n",
       "      <td>0.277960</td>\n",
       "      <td>0.341241</td>\n",
       "      <td>0.344504</td>\n",
       "      <td>0.542990</td>\n",
       "      <td>0.377536</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8378</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0.433457</td>\n",
       "      <td>0.624660</td>\n",
       "      <td>0.433457</td>\n",
       "      <td>0.624660</td>\n",
       "      <td>0.566543</td>\n",
       "      <td>0.375340</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4225</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0.463775</td>\n",
       "      <td>0.594534</td>\n",
       "      <td>0.463775</td>\n",
       "      <td>0.594534</td>\n",
       "      <td>0.536225</td>\n",
       "      <td>0.405466</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3066</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0.121112</td>\n",
       "      <td>0.356151</td>\n",
       "      <td>0.878888</td>\n",
       "      <td>0.643849</td>\n",
       "      <td>0.878888</td>\n",
       "      <td>0.643849</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14398</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.071045</td>\n",
       "      <td>0.269898</td>\n",
       "      <td>0.928955</td>\n",
       "      <td>0.730102</td>\n",
       "      <td>0.928955</td>\n",
       "      <td>0.730102</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17874</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.613035</td>\n",
       "      <td>0.620146</td>\n",
       "      <td>0.386965</td>\n",
       "      <td>0.379854</td>\n",
       "      <td>0.613035</td>\n",
       "      <td>0.620146</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25132</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0.167880</td>\n",
       "      <td>0.228945</td>\n",
       "      <td>0.832120</td>\n",
       "      <td>0.771055</td>\n",
       "      <td>0.832120</td>\n",
       "      <td>0.771055</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17848</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.802908</td>\n",
       "      <td>0.765155</td>\n",
       "      <td>0.197092</td>\n",
       "      <td>0.234845</td>\n",
       "      <td>0.802908</td>\n",
       "      <td>0.765155</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       t1  t2  t3   t1_cont   t1_attn   t2_cont   t2_attn   t3_cont   t3_attn\n",
       "3764   2   2   1   0.383158  0.659369  0.383158  0.659369  0.616842  0.340631\n",
       "27181  1   1   0   0.527001  0.622294  0.527001  0.622294  0.472999  0.377706\n",
       "14482  2   1   0   0.115770  0.277960  0.341241  0.344504  0.542990  0.377536\n",
       "8378   2   2   1   0.433457  0.624660  0.433457  0.624660  0.566543  0.375340\n",
       "4225   1   1   2   0.463775  0.594534  0.463775  0.594534  0.536225  0.405466\n",
       "3066   0   2   2   0.121112  0.356151  0.878888  0.643849  0.878888  0.643849\n",
       "14398  2   1   1   0.071045  0.269898  0.928955  0.730102  0.928955  0.730102\n",
       "17874  0   1   0   0.613035  0.620146  0.386965  0.379854  0.613035  0.620146\n",
       "25132  0   2   2   0.167880  0.228945  0.832120  0.771055  0.832120  0.771055\n",
       "17848  0   1   0   0.802908  0.765155  0.197092  0.234845  0.802908  0.765155"
      ]
     },
     "execution_count": 376,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds_df3[(preds_df3['actual']==1) & ((preds_df3['t2']!=preds_df3['t3']) | (preds_df3['t1']!=preds_df3['t2']))][['t1', 't2', 't3','t1_cont', 't1_attn', 't2_cont', 't2_attn', 't3_cont','t3_attn']].sample(10)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Distribution of actual T1 attribution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 377,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAP20lEQVR4nO3df4ylVX3H8fenqNhWKugOhC5Lh9o1EU1czARp/KMopiIkriZiINGuhrjGQqPRNK72D+0PkrWtkppa2rUQV6PC1h9lI7SWIsbaFHRQRJYtccUtjLthR0HUGGnBb/+YZ3XYndn7zI87d+fM+5Xc3Oc599w73znZ+czZc597JlWFJKktvzLqAiRJy89wl6QGGe6S1CDDXZIaZLhLUoOeMuoCANatW1fj4+OjLkOSVpU777zz+1U1Ntdjx0W4j4+PMzk5OeoyJGlVSfI/8z3msowkNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXouPiE6mo1vu2mXv32b794yJVI0pMNnLkneXqSryb5ZpI9Sf60az8ryR1Jvp3khiRP69pP7M73dY+PD/dbkCQdqc+yzGPAy6rqhcAm4MIk5wHvB66uqo3AI8DlXf/LgUeq6neAq7t+kqQVNDDca8ZPutOndrcCXgZ8umvfCby6O97cndM9fkGSLFvFkqSBer2hmuSEJHcBh4BbgO8AP6yqx7suU8D67ng98CBA9/ijwLPneM2tSSaTTE5PTy/tu5AkPUmvcK+qJ6pqE3AGcC7wvLm6dfdzzdLrqIaqHVU1UVUTY2NzbkcsSVqkBV0KWVU/BL4EnAecnOTw1TZnAAe64ylgA0D3+DOBh5ejWElSP32ulhlLcnJ3/KvAy4G9wG3Aa7tuW4Abu+Pd3Tnd41+sqqNm7pKk4elznfvpwM4kJzDzy2BXVX0+yb3A9Un+AvgGcG3X/1rg40n2MTNjv3QIdUuSjmFguFfV3cA5c7Tfz8z6+5HtPwMuWZbqJEmL4vYDktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBffZzX3PGt9006hIkaUmcuUtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lq0MBwT7IhyW1J9ibZk+RtXfv7knwvyV3d7aJZz3l3kn1J7kvyimF+A5Kko/XZW+Zx4J1V9fUkJwF3Jrmle+zqqvrr2Z2TnA1cCjwf+E3g35M8t6qeWM7CJUnzGzhzr6qDVfX17vjHwF5g/TGeshm4vqoeq6rvAvuAc5ejWElSPwtac08yDpwD3NE1XZnk7iTXJTmla1sPPDjraVPM8csgydYkk0kmp6enF1y4JGl+vcM9yTOAzwBvr6ofAdcAzwE2AQeBDxzuOsfT66iGqh1VNVFVE2NjYwsuXJI0v17hnuSpzAT7J6rqswBV9VBVPVFVPwc+wi+XXqaADbOefgZwYPlKliQN0udqmQDXAnur6oOz2k+f1e01wD3d8W7g0iQnJjkL2Ah8dflKliQN0udqmZcAbwC+leSuru09wGVJNjGz5LIfeAtAVe1Jsgu4l5krba7wShlJWlkDw72qvsLc6+g3H+M5VwFXLaEuSdIS+AlVSWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDWoz66QWqLxbTf16rd/+8VDrkTSWuHMXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNGhjuSTYkuS3J3iR7kryta39WkluSfLu7P6VrT5IPJdmX5O4kLxr2NyFJerI+M/fHgXdW1fOA84ArkpwNbANuraqNwK3dOcArgY3dbStwzbJXLUk6poHhXlUHq+rr3fGPgb3AemAzsLPrthN4dXe8GfhYzbgdODnJ6cteuSRpXgtac08yDpwD3AGcVlUHYeYXAHBq12098OCsp011bUe+1tYkk0kmp6enF165JGlevcM9yTOAzwBvr6ofHavrHG11VEPVjqqaqKqJsbGxvmVIknroFe5JnspMsH+iqj7bNT90eLmluz/UtU8BG2Y9/QzgwPKUK0nqo8/VMgGuBfZW1QdnPbQb2NIdbwFunNX+B91VM+cBjx5evpEkrYw+f0P1JcAbgG8luatrew+wHdiV5HLgAeCS7rGbgYuAfcBPgTcta8WSpIEGhntVfYW519EBLpijfwFXLLGuoej7h6olabXzE6qS1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJalCfXSG1QvpubLZ/+8VDrkTSaufMXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJatDAcE9yXZJDSe6Z1fa+JN9Lcld3u2jWY+9Osi/JfUleMazCJUnz6zNz/yhw4RztV1fVpu52M0CSs4FLged3z/m7JCcsV7GSpH4GhntVfRl4uOfrbQaur6rHquq7wD7g3CXUJ0lahKWsuV+Z5O5u2eaUrm098OCsPlNdmyRpBS023K8BngNsAg4CH+jaM0ffmusFkmxNMplkcnp6epFlSJLmsqhwr6qHquqJqvo58BF+ufQyBWyY1fUM4MA8r7GjqiaqamJsbGwxZUiS5rGocE9y+qzT1wCHr6TZDVya5MQkZwEbga8urURJ0kIN/DN7ST4FnA+sSzIFvBc4P8kmZpZc9gNvAaiqPUl2AfcCjwNXVNUTwyldkjSfgeFeVZfN0XztMfpfBVy1lKIkSUvjJ1QlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0a+Mc6JLVhfNtNvfrt337xkCvRSnDmLkkNcuYurXJ9Z+RaW5y5S1KDDHdJapDhLkkNMtwlqUGGuyQ1yKtlpOOUV8FoKQbO3JNcl+RQkntmtT0ryS1Jvt3dn9K1J8mHkuxLcneSFw2zeEnS3PrM3D8K/C3wsVlt24Bbq2p7km3d+buAVwIbu9uLgWu6e0mrxEL+x+CnWY9fA2fuVfVl4OEjmjcDO7vjncCrZ7V/rGbcDpyc5PTlKlaS1M9i19xPq6qDAFV1MMmpXft64MFZ/aa6toNHvkCSrcBWgDPPPHORZUirj2vpWgnLfbVM5miruTpW1Y6qmqiqibGxsWUuQ5LWtsWG+0OHl1u6+0Nd+xSwYVa/M4ADiy9PkrQYi12W2Q1sAbZ39zfOar8yyfXMvJH66OHlGy0ft26VNMjAcE/yKeB8YF2SKeC9zIT6riSXAw8Al3TdbwYuAvYBPwXeNISaJUkDDAz3qrpsnocumKNvAVcstShJ0tK4/YAkNchwl6QGubeMtEzW4vXrvrl//HLmLkkNMtwlqUGGuyQ1yDV3SUPn2vzKc+YuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkN8hOq0gBrcbdHrX6Gu9YkA1utc1lGkhpkuEtSg1yWkXTccPfI5ePMXZIaZLhLUoMMd0lqkOEuSQ1a0huqSfYDPwaeAB6vqokkzwJuAMaB/cDrquqRpZUpSVqI5Zi5v7SqNlXVRHe+Dbi1qjYCt3bnkqQVNIxLITcD53fHO4EvAe8awteRjuInT6UZS525F/BvSe5MsrVrO62qDgJ096fO9cQkW5NMJpmcnp5eYhmSpNmWOnN/SVUdSHIqcEuS/+77xKraAewAmJiYqCXWIUmaZUkz96o60N0fAj4HnAs8lOR0gO7+0FKLlCQtzKLDPcmvJznp8DHw+8A9wG5gS9dtC3DjUouUJC3MUpZlTgM+l+Tw63yyqv41ydeAXUkuBx4ALll6mZKkhVh0uFfV/cAL52j/AXDBUorS2uFGUdJwrPpdIb30TZKOturDXWuDv8SlhTHcJa06LucN5sZhktQgw12SGmS4S1KDDHdJapDhLkkN8moZDYWXLkqj5cxdkhrkzL1hC5k9r+XrgaUWOXOXpAY5c9eCuJYurQ7O3CWpQc7cJa15Le5V48xdkhpkuEtSg1yWkdSstXwBgOEuYG3/EEgtcllGkhpkuEtSgwx3SWqQa+6S1NNq2q/JmbskNchwl6QGDW1ZJsmFwN8AJwD/WFXbh/W1JOl4M+otDYYyc09yAvBh4JXA2cBlSc4exteSJB1tWMsy5wL7qur+qvpf4Hpg85C+liTpCMNallkPPDjrfAp48ewOSbYCW7vTnyS5rzteB3x/SHWtdo7N/Byb+Tk2xzbS8cn7l/T035rvgWGFe+ZoqyedVO0Adhz1xGSyqiaGVNeq5tjMz7GZn2NzbK2Oz7CWZaaADbPOzwAODOlrSZKOMKxw/xqwMclZSZ4GXArsHtLXkiQdYSjLMlX1eJIrgS8wcynkdVW1p+fTj1qq0S84NvNzbObn2Bxbk+OTqhrcS5K0qvgJVUlqkOEuSQ0aSbgnuTDJfUn2Jdk2x+MnJrmhe/yOJOMrX+Xo9BifdyS5N8ndSW5NMu+1rq0ZNDaz+r02SSVp7hK3+fQZmySv6/7t7EnyyZWucVR6/EydmeS2JN/ofq4uGkWdy6qqVvTGzBus3wF+G3ga8E3g7CP6/CHw993xpcANK13nqG49x+elwK91x29dK+PTZ2y6ficBXwZuByZGXffxMjbARuAbwCnd+amjrvs4GpsdwFu747OB/aOue6m3Uczc+2xNsBnY2R1/GrggyVwfjGrRwPGpqtuq6qfd6e3MfI5gLei7rcWfA38J/GwlixuxPmPzZuDDVfUIQFUdWuEaR6XP2BTwG93xM2ngczmjCPe5tiZYP1+fqnoceBR49opUN3p9xme2y4F/GWpFx4+BY5PkHGBDVX1+JQs7DvT5d/Nc4LlJ/jPJ7d3OrWtBn7F5H/D6JFPAzcAfrUxpwzOKv8Q0cGuCnn1a1ft7T/J6YAL4vaFWdPw45tgk+RXgauCNK1XQcaTPv5unMLM0cz4z/9v7jyQvqKofDrm2UeszNpcBH62qDyT5XeDj3dj8fPjlDccoZu59tib4RZ8kT2Hmv0kPr0h1o9dr64YkLwf+BHhVVT22QrWN2qCxOQl4AfClJPuB84Dda+RN1b4/VzdW1f9V1XeB+5gJ+9b1GZvLgV0AVfVfwNOZ2VBs1RpFuPfZmmA3sKU7fi3wxere6VgDBo5Pt/TwD8wE+1pZN4UBY1NVj1bVuqoar6pxZt6PeFVVTY6m3BXV5+fqn5l5M54k65hZprl/RascjT5j8wBwAUCS5zET7tMrWuUyW/Fw79bQD29NsBfYVVV7kvxZkld13a4Fnp1kH/AOYN5L3lrTc3z+CngG8E9J7kqyJvbt6Tk2a1LPsfkC8IMk9wK3AX9cVT8YTcUrp+fYvBN4c5JvAp8C3rjaJ5RuPyBJDfITqpLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNej/AfOWIx3nUDBnAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "_=plt.hist(preds_df3[(preds_df3['actual']==1) & ((preds_df3['t2']!=preds_df3['t3']) | (preds_df3['t1']!=preds_df3['t2']))]['t1_cont'], bins=30)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Plot Estimated Attribution vs. Actual Attribution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 378,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds_df3 = preds_df3[(preds_df3['t1']!=preds_df3['t2']) | (preds_df3['t1']!=preds_df3['t3'])] \n",
    "preds_df3 = preds_df3[(preds_df3['actual']==1)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 379,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD5CAYAAAA3Os7hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAPIElEQVR4nO3db4xcZ3mG8euOjYGSAAVvK7Ad1gKn1KIRqBuDSlsCBMlRKrtSaeuIVnFFcVGxqARCmBalyFWlNFTlA3JV3DYIUJEJCOiWmFpqAKECod5AmmAHKysT8MqV2ITwT5Q4Lk8/7CQaTWZ3zjqzXvvN9ZNGnvc9zznn2X2t22fPzoxTVUiSLn6XrHYDkqTxMNAlqREGuiQ1wkCXpEYY6JLUiLWrdeL169fX5OTkap1eki5Kd9555wNVNTFs26oF+uTkJDMzM6t1ekm6KCX59mLbvOUiSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNWLV3ikq6sE3uu221W2jW/TddtyLH9QpdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEZ0CPcn2JCeSzCbZt0jN7yU5nuRYko+Ot01J0igjPz43yRrgAPA6YA44mmS6qo731WwB3gW8sqoeSvILK9WwJGm4Llfo24DZqjpZVWeAQ8DOgZo3AQeq6iGAqvrueNuUJI3SJdA3AKf6xnO9uX5XAFck+VKSO5JsH3agJHuSzCSZmZ+fP7eOJUlDdQn0DJmrgfFaYAtwNXA98E9Jnv24naoOVtVUVU1NTEwst1dJ0hK6BPocsKlvvBE4PaTmX6vqkar6FnCChYCXJJ0nXQL9KLAlyeYk64BdwPRAzaeBVwMkWc/CLZiT42xUkrS0kYFeVWeBvcAR4F7g1qo6lmR/kh29siPAg0mOA58H3lFVD65U05Kkxxv5skWAqjoMHB6Yu7HveQFv6z0kSavAd4pKUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1IhOgZ5ke5ITSWaT7BuyfXeS+SR39R5/PP5WJUlLWTuqIMka4ADwOmAOOJpkuqqOD5R+rKr2rkCPasDkvttWu4Vm3X/Tdavdgi4QXa7QtwGzVXWyqs4Ah4CdK9uWJGm5ugT6BuBU33iuNzfod5LcneQTSTYNO1CSPUlmkszMz8+fQ7uSpMV0CfQMmauB8b8Bk1V1JfAfwIeGHaiqDlbVVFVNTUxMLK9TSdKSugT6HNB/xb0RON1fUFUPVtXDveE/Ar86nvYkSV11CfSjwJYkm5OsA3YB0/0FSZ7XN9wB3Du+FiVJXYx8lUtVnU2yFzgCrAFuqapjSfYDM1U1Dbw1yQ7gLPA9YPcK9ixJGmJkoANU1WHg8MDcjX3P3wW8a7ytSZKWw3eKSlIjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRnQK9CTbk5xIMptk3xJ1r09SSabG16IkqYuRgZ5kDXAAuBbYClyfZOuQusuAtwJfHXeTkqTRulyhbwNmq+pkVZ0BDgE7h9T9FXAz8NMx9idJ6qhLoG8ATvWN53pzj0nyMmBTVX1mqQMl2ZNkJsnM/Pz8spuVJC2uS6BnyFw9tjG5BHgf8PZRB6qqg1U1VVVTExMT3buUJI3UJdDngE19443A6b7xZcBLgC8kuR94BTDtL0Yl6fzqEuhHgS1JNidZB+wCph/dWFU/qKr1VTVZVZPAHcCOqppZkY4lSUONDPSqOgvsBY4A9wK3VtWxJPuT7FjpBiVJ3aztUlRVh4HDA3M3LlJ79RNvS5K0XL5TVJIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjOr31/0Izue+21W6hWfffdN1qtyDpHHmFLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGdAr0JNuTnEgym2TfkO1vTnJPkruS/GeSreNvVZK0lJGBnmQNcAC4FtgKXD8ksD9aVb9SVS8Fbgb+buydSpKW1OUKfRswW1Unq+oMcAjY2V9QVT/sGz4DqPG1KEnqosvnoW8ATvWN54CXDxYleQvwNmAd8JphB0qyB9gDcPnlly+3V0nSErpcoWfI3OOuwKvqQFW9EHgn8O5hB6qqg1U1VVVTExMTy+tUkrSkLoE+B2zqG28ETi9Rfwj47SfSlCRp+boE+lFgS5LNSdYBu4Dp/oIkW/qG1wH3ja9FSVIXI++hV9XZJHuBI8Aa4JaqOpZkPzBTVdPA3iTXAI8ADwE3rGTTkqTH6/SfRFfVYeDwwNyNfc//bMx9SZKWyXeKSlIjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRnQK9CTbk5xIMptk35Dtb0tyPMndSW5P8oLxtypJWsrIQE+yBjgAXAtsBa5PsnWg7OvAVFVdCXwCuHncjUqSltblCn0bMFtVJ6vqDHAI2NlfUFWfr6qf9IZ3ABvH26YkaZQugb4BONU3nuvNLeaNwGefSFOSpOVb26EmQ+ZqaGHyB8AU8KpFtu8B9gBcfvnlHVuUJHXR5Qp9DtjUN94InB4sSnIN8BfAjqp6eNiBqupgVU1V1dTExMS59CtJWkSXQD8KbEmyOck6YBcw3V+Q5GXAB1gI8++Ov01J0igjA72qzgJ7gSPAvcCtVXUsyf4kO3pl7wUuBT6e5K4k04scTpK0QrrcQ6eqDgOHB+Zu7Ht+zZj7kiQtk+8UlaRGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRnQI9yfYkJ5LMJtk3ZPtvJvlakrNJXj/+NiVJo4wM9CRrgAPAtcBW4PokWwfKvgPsBj467gYlSd2s7VCzDZitqpMASQ4BO4HjjxZU1f29bT9bgR4lSR10ueWyATjVN57rzS1bkj1JZpLMzM/Pn8shJEmL6BLoGTJX53KyqjpYVVNVNTUxMXEuh5AkLaJLoM8Bm/rGG4HTK9OOJOlcdQn0o8CWJJuTrAN2AdMr25YkablGBnpVnQX2AkeAe4Fbq+pYkv1JdgAkuSrJHPC7wAeSHFvJpiVJj9flVS5U1WHg8MDcjX3Pj7JwK0aStEp8p6gkNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjegU6Em2JzmRZDbJviHbn5rkY73tX00yOe5GJUlLGxnoSdYAB4Brga3A9Um2DpS9EXioql4EvA/4m3E3KklaWpcr9G3AbFWdrKozwCFg50DNTuBDveefAF6bJONrU5I0ytoONRuAU33jOeDli9VU1dkkPwCeCzzQX5RkD7CnN/xxkhPn0vRFaD0D34sLVfzZCi6i9QLXrOfJtGYvWGxDl0AfdqVd51BDVR0EDnY4Z1OSzFTV1Gr3oW5cr4uPa7agyy2XOWBT33gjcHqxmiRrgWcB3xtHg5KkbroE+lFgS5LNSdYBu4DpgZpp4Ibe89cDn6uqx12hS5JWzshbLr174nuBI8Aa4JaqOpZkPzBTVdPAPwMfSTLLwpX5rpVs+iL0pLvNdJFzvS4+rhkQL6QlqQ2+U1SSGmGgS1IjDHRJaoSBvogkz07yp33jf0/y/SSfOY89/Pn5OlcL+tcsyUuTfCXJsSR3J/n989SDa7YMA2v2giR3Jrmrt25vPp/nb4G/FF1E7wPGPlNVL+mNXwv8HPAnVfVb56mHH1fVpefjXC3oX7MkVwBVVfcleT5wJ/DLVfX9Fe7BNVuGgTVbx0ImPZzkUuAbwK9V1eD7Xlbk/Ct1jvPJK/TF3QS8sHe18N6quh34UZcdk1yV5MtJ/jvJfyW5LMnTknwwyT1Jvp7k1b3a3Uk+2fsJ4L4kN/fmbwKe3jv/v6zYV9mWx9YMeFNV3QfQC4TvAhOL7eiarZr+Nfvrqnq4N/9URuRT71Ngv9Zbs9t7c89J8uneT2V3JLmyN/+eJLck+UKSk0neOnj+JO9dmS/xPKoqH0MewCTwjYG5q1n413yp/dYBJ4GreuNnsvB6/7cDH+zNvRj4DvA0YHev/lm98beBTb26H6/29+Fiegxbs978NuBe4BLX7MJ6DK4ZC+84vxv4CfCWJfabYOHzozb3xs/p/fl+4C97z18D3NV7/h7gyyz8Q7EeeBB4ymJ/Zy7Wh1fo4/dLwP9U1VGAqvphVZ0Ffh34SG/umyyEwBW9fW6vqh9U1U+B4yzx4TtaniTPY+H7/kdV9bNFylyzC0RVnaqqK4EXATck+cVFSl8BfLGqvtXb79GPGulfs88Bz03yrN6226rq4ap6gIWf2BY79kXLQB+/MOSDyRj+AWaPerjv+f/R7UPTNEKSZwK3Ae+uqjuWKsU1u6DUwm2yY8BvLFKynDV7tK75NTPQF/cj4LJz2O+bwPOTXAXQuxe7Fvgi8Ibe3BXA5cCojw9+JMlTzqGHJ6vH1qz3C7ZPAR+uqo+P2M81Wz39a7YxydN7z38eeCWLf7+/ArwqyeZe/XN68/1rdjXwQFX9sMv5W9Dcv1DjUlUPJvlSkm8An2XhR7wXA5cmmQPeWFVHhux3pvcSuff3/nL+L3AN8PfAPyS5BzgL7K6F3+Yv1cZB4O4kX6uqN4z1C2zQwJo9g4VPBn1ukt29kt1VddeQ/VyzVTKwZo8Aa5IUC1faf1tV9yyy33wW/n+FTya5hIVbKK9j4V75B5M8eh/+hmH7L3L+z1bVO8b2xa0CX7YoSY3wloskNcJbLk9Akk8Bmwem3znsVowuDK7ZxSfJV1l4uWG/P1zsdsyTmbdcJKkR3nKRpEYY6JLUCANdkhphoEtSI/4f+pato+2Cb0gAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "_=plt.bar(['t1_cont', 't2_cont', 't3_cont'],preds_df3[['t1_cont', 't2_cont', 't3_cont']].mean())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 380,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD5CAYAAAA3Os7hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAN6UlEQVR4nO3df6jd913H8eeridmvduqWi2xJ2hu2iMatdHjNpqJua4WEaSIYNYVJK8U4XOhkDta5UTAyqK04BKMunWNzbGRdYXBtUyPO9g+HLblZu7ZpCQ2xmmuEpdtaV6dN497+cU/K8fTce75Jz9nN/fT5gEvP9/v9nO/93PsJz3zzvfecpqqQJK18lyz3BCRJ42HQJakRBl2SGmHQJakRBl2SGrF6uT7x2rVra3p6erk+vSStSEeOHHmqqqaGHVu2oE9PTzM3N7dcn16SVqQk/7rYMW+5SFIjDLokNcKgS1IjDLokNcKgS1IjDLokNcKgS1IjDLokNcKgS1Ijlu2VopIubtM33b3cU2jWk7e8ZyLn9Qpdkhph0CWpEQZdkhph0CWpEQZdkhph0CWpEQZdkhph0CWpEQZdkhph0CWpEQZdkhph0CWpEZ3enCvJVuDPgFXAp6rqloHj1wO3Af/e2/XnVfWpMc5TK5xv9DQ5k3qjJ608I4OeZBWwD/hFYB44nGS2qh4bGPrFqtozgTlKkjrocstlC3C8qk5U1RngALBjstOSJJ2vLkFfB5zs257v7Rv0q0keTnJnkg3DTpRkd5K5JHOnT5++gOlKkhbTJegZsq8Gtv8WmK6qK4F/AD477ERVtb+qZqpqZmpq6vxmKklaUpegzwP9V9zrgVP9A6rqm1X1XG/zduAnxzM9SVJXXYJ+GNiUZGOSNcAuYLZ/QJI39G1uBx4f3xQlSV2M/C2XqjqbZA9wiIVfW/x0VR1NsheYq6pZ4MYk24GzwLeA6yc4Z0nSEJ1+D72qDgIHB/bd3Pf4I8BHxjs1SdL58JWiktQIgy5JjTDoktQIgy5JjTDoktQIgy5JjTDoktQIgy5Jjej0wqKLjf+zhMnxf5YgrVxeoUtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDWiU9CTbE1yLMnxJDctMW5nkkoyM74pSpK6GBn0JKuAfcA2YDNwbZLNQ8ZdBtwIPDDuSUqSRutyhb4FOF5VJ6rqDHAA2DFk3B8BtwL/M8b5SZI66hL0dcDJvu353r4XJHkbsKGq7lrqREl2J5lLMnf69OnznqwkaXFdgp4h++qFg8klwCeA3x91oqraX1UzVTUzNTXVfZaSpJG6BH0e2NC3vR441bd9GfAW4L4kTwLvAGb9wagkfX91CfphYFOSjUnWALuA2XMHq+qZqlpbVdNVNQ3cD2yvqrmJzFiSNNTIoFfVWWAPcAh4HLijqo4m2Ztk+6QnKEnqZnWXQVV1EDg4sO/mRca+86VPS5J0vnylqCQ1wqBLUiMMuiQ1wqBLUiMMuiQ1wqBLUiMMuiQ1wqBLUiMMuiQ1wqBLUiMMuiQ1wqBLUiMMuiQ1wqBLUiMMuiQ1wqBLUiMMuiQ1wqBLUiMMuiQ1wqBLUiMMuiQ1wqBLUiMMuiQ1wqBLUiMMuiQ1wqBLUiMMuiQ1wqBLUiMMuiQ1olPQk2xNcizJ8SQ3DTn+viSPJHkoyT8l2Tz+qUqSljIy6ElWAfuAbcBm4Nohwf5CVb21qq4CbgX+dOwzlSQtqcsV+hbgeFWdqKozwAFgR/+AqvrPvs3XADW+KUqSuljdYcw64GTf9jzw9sFBSd4PfBBYA7x72ImS7AZ2A1x++eXnO1dJ0hK6XKFnyL4XXYFX1b6qehPwYeBjw05UVfuraqaqZqamps5vppKkJXUJ+jywoW97PXBqifEHgF95KZOSJJ2/LkE/DGxKsjHJGmAXMNs/IMmmvs33AE+Mb4qSpC5G3kOvqrNJ9gCHgFXAp6vqaJK9wFxVzQJ7klwDPA98G7hukpOWJL1Ylx+KUlUHgYMD+27ue/yBMc9LknSefKWoJDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDWiU9CTbE1yLMnxJDcNOf7BJI8leTjJV5JcMf6pSpKWMjLoSVYB+4BtwGbg2iSbB4Y9CMxU1ZXAncCt456oJGlpXa7QtwDHq+pEVZ0BDgA7+gdU1b1V9d3e5v3A+vFOU5I0SpegrwNO9m3P9/Yt5gbgnmEHkuxOMpdk7vTp091nKUkaqUvQM2RfDR2YvBeYAW4bdryq9lfVTFXNTE1NdZ+lJGmk1R3GzAMb+rbXA6cGByW5Bvgo8AtV9dx4pidJ6qrLFfphYFOSjUnWALuA2f4BSd4GfBLYXlXfGP80JUmjjAx6VZ0F9gCHgMeBO6rqaJK9Sbb3ht0GXAp8KclDSWYXOZ0kaUK63HKhqg4CBwf23dz3+Joxz0uSdJ58pagkNcKgS1IjDLokNcKgS1IjDLokNcKgS1IjDLokNcKgS1IjDLokNcKgS1IjDLokNcKgS1IjDLokNcKgS1IjDLokNcKgS1IjDLokNcKgS1IjDLokNcKgS1IjDLokNcKgS1IjDLokNcKgS1IjDLokNcKgS1IjDLokNcKgS1IjDLokNaJT0JNsTXIsyfEkNw05/vNJvpbkbJKd45+mJGmUkUFPsgrYB2wDNgPXJtk8MOzfgOuBL4x7gpKkblZ3GLMFOF5VJwCSHAB2AI+dG1BVT/aOfW8Cc5QkddDllss64GTf9nxvnyTpItIl6Bmyry7kkyXZnWQuydzp06cv5BSSpEV0Cfo8sKFvez1w6kI+WVXtr6qZqpqZmpq6kFNIkhbRJeiHgU1JNiZZA+wCZic7LUnS+RoZ9Ko6C+wBDgGPA3dU1dEke5NsB0jyU0nmgV8DPpnk6CQnLUl6sS6/5UJVHQQODuy7ue/xYRZuxUiSlomvFJWkRhh0SWqEQZekRhh0SWqEQZekRhh0SWqEQZekRhh0SWqEQZekRhh0SWqEQZekRhh0SWqEQZekRhh0SWqEQZekRhh0SWqEQZekRhh0SWqEQZekRhh0SWqEQZekRhh0SWqEQZekRhh0SWqEQZekRhh0SWqEQZekRhh0SWqEQZekRhh0SWpEp6An2ZrkWJLjSW4acvwVSb7YO/5AkulxT1SStLSRQU+yCtgHbAM2A9cm2Tww7Abg21X1ZuATwB+Pe6KSpKV1uULfAhyvqhNVdQY4AOwYGLMD+Gzv8Z3A1UkyvmlKkkZZ3WHMOuBk3/Y88PbFxlTV2STPAK8HnuoflGQ3sLu3+WySYxcy6RVoLQPfi4tV/LcVrKD1Ates5+W0ZlcsdqBL0IddadcFjKGq9gP7O3zOpiSZq6qZ5Z6HunG9Vh7XbEGXWy7zwIa+7fXAqcXGJFkN/CDwrXFMUJLUTZegHwY2JdmYZA2wC5gdGDMLXNd7vBP4x6p60RW6JGlyRt5y6d0T3wMcAlYBn66qo0n2AnNVNQv8NfC5JMdZuDLfNclJr0Avu9tMK5zrtfK4ZkC8kJakNvhKUUlqhEGXpEYYdElqhEFfQpIfSvK7fdt/l+TpJHe9hHNen+SNfdu/l+TVL3WuWtC/ZkmuSvLPSY4meTjJb1zgOV2zCRpYsyuSHEnyUG/d3neB5/x/a5TkD8Y134uZPxRdQu9Nxu6qqrf0tq8GXg38TlX90gWe8z7gQ1U119t+EpipqhXzKreLWf+aJflRoKrqiV6QjwA/XlVPn+c578M1m5iBNVvDQpeeS3Ip8CjwM1U1+NqXUed8kr41SvJsVV063plffLxCX9otwJt6Vwu3VdVXgO90eWKSm5McTvJokv1ZsBOYAT7fO+cHgDcC9ya5t/e8Z5N8PMnXk9yf5Ecm9cU16oU1A367qp4A6AXhG8DUYk90zZZN/5p9vKqe6+1/BSMaleQvk8z1rub/sLfvRvrWKMktwKt66/f5JNNJHk9ye+95f5/kVRP8+r5/qsqPRT6AaeDRgX3vZOFqYtRzX9f3+HPAL/ce38fClcO5Y08Ca/u2q2/srcDHlvv7sJI+hq1Zb/8W4HHgEtfs4voYXDMWXnX+MPBd4P0jnvu63n9X9dbpykXW6NmBz3cWuKq3fQfw3uX+Pozjwyv0yXlX773hHwHeDfxEx+edAc7doz/Cwh8+vQRJ3sBCoH+rqr63xFDX7CJQVSer6krgzcB1I/7F8+tJvgY8yMJ6Db6192L+paoe6j1uZs0M+gQkeSXwF8DOqnorcDvwyo5Pf756lw3A/9LtDdS0iCSvBe5m4ar5/iXGuWYXmVq4TXYU+Llhx5NsBD4EXN37C+Buuq/Zc32Pm1kzg7607wCXXcDzzv2heqr3g52dS5zzQj+Hhnvh+9n7AduXgb+pqi+NeJ5rtnz612z9ufvZSX4Y+FlgsbfZfi3wX8Azvav4bcPO2fN8kh8Y98QvNk38rTQpVfXNJF9N8ihwD/AO4MeAS5PMAzdU1aEhz3s6ye3AIyzcyzvcd/gzwF8l+W/gp1l4D4p7kvxHVb1rol/Qy8DAmr2GhXcHfX2S63tDru/7p3b/81yzZTKwZs8Dq5IUC2/L/SdV9cgiz/t6kgdZuIo/AXy17/DgGu0HHu7dnvnoJL+e5eSvLUpSI7zlIkmN8JbLS5Tky8DGgd0fHnYrRhcH12zlSfIAC7+X3u83F7sd83LlLRdJaoS3XCSpEQZdkhph0CWpEQZdkhrxfzkVS9Adul+GAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "_=plt.bar(['t1_attn', 't2_attn', 't3_attn'],preds_df3[['t1_attn', 't2_attn', 't3_attn']].mean())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 381,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1a89f8ca10>]"
      ]
     },
     "execution_count": 381,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deXxU1d3H8c+ZySSZBMiETSVAWUQiixINioK1FBVRwYgiohZaBawWtUJ5WCKymyjyqC+3p2jrUlBQ0YC4hCq4gGABw2JAVlEJLhRIgBCynuePZCYzyUxyJ5nJbL/369VXM/fe3Pk5wDcn555Faa0RQggR+kyBLkAIIYRvSKALIUSYkEAXQogwIYEuhBBhQgJdCCHCRFSg3rh169a6U6dOgXp7IYQISVu2bPmv1rqNu3MBC/ROnTqxefPmQL29EEKEJKXU957OSZeLEEKECQl0IYQIExLoQggRJiTQhRAiTEigCyFEmJBAF0KIMCGBLoQQYUICXQghwkTAJhYJIUS4yMrJY0H2bg7nF9HOZmXy4O6kpSQ1eR0S6EII0QhZOXlMe2cHRaXlAOTlFzHtnR0ATR7qEuhCiIjkq1b1guzdjjC3KyotZ0H2bgl0IYRoDCNB7ctW9eH8Iq+O+5M8FBVChA17UOflF6GpDuqsnDyX6+pqVXurnc3q1XF/kkAXQoQNT0E9a2WuyzFftqonD+6O1WJ2OWa1mJk8uLvX92osCXQhRNjwFMj5RaUurXRftqrTUpLIGN6bJJsVBSTZrGQM7y2jXIQQojHa2azkeQh154eUkwd3d+lDh8a1qtNSkgIS4DVJC10IETbqCmTn1ntTtaqzcvLon7mGzlPfp3/mmlp9+b4mLXQhRNhIS0li9nu5HD9dWutcze4UX7eqa46uGZjchuVb8pp0fLq00IUQYeX6C85xe3xgstttOH3C3eiaJRt/8NlIGqMk0IUQYWXtt0e8Ou4Ls1bm1gpv7eFab0fSlJeXU1pa+zcOd6TLRQgRVvw50cddt8qqbT+RX2QscMH4SBqtNatWrWL69On86U9/YuLEifV+j7TQhRBhxV8Tfdx1qyze+INXYW50JM0XX3zBgAEDGDZsGN988w2PPvooBQUF9X6fBLoQIqz4a6KPu0lLRhkdSbNt2zauv/56fvvb3/Lll186jp85c4YtW7bU+z7S5SKECCv2wPT1craN6bL5LvP6Os/v37+fRx55hDfeeAOtq3vfLRYL9957L+np6bRt27be95FAF0KELE8LcTVmSKKne9Y1aakuiXEWj+d+/vln5s6dy6JFiygrK3McV0rxhz/8gdmzZ9OpUyfD7yWBLoQISY1dMdEe3Hn5RZiVolxrbFYLhSVllJZrl3tu/v4YhcVl9dyxNotZMXNoT4/nX3nlFZ5//nmXY8OGDWP+/Pn06tXL6/eTPnQhREhqzIqJzg84Acqrujnyi0odYe58T28ffjpqvOXCOn+43H///Zx99tkAXHHFFaxfv54VK1Y0KMxBWuhCiBDVmOGJjXnAaVRinMUR5mVlZbz88stcfvnl9OxZ3WKPj4/nueeeIzY2liFDhqCUatR7GmqhK6WuVUrtVkrtU0pNdXO+o1JqrVIqRym1XSl1XaOqEkKIengahmiLs9S7foq/N58wmyq7WrTWvPXWW/Ts2ZPx48eTnp5e69rhw4dz3XXXNTrMwUCgK6XMwHPAEKAHMEop1aPGZQ8Db2qtU4DbgOcRQgg/cjc80WJWnDpTVu8GFwlWzw8qGysxzsLCERfS7Ogu+vbty6233sqePXsAWLFiBRs2bPDbexvpcrkE2Ke1PgCglFoK3AjsdLpGAy2qvk4ADvuySCGEqMnd8MTjhcWcLq1wua7m/p4PZ+1oUH94fZ4a2Ye0lCQ2bdrEtMlj+OSTT1zOt2jRgilTpnDBBRf4/L3tjAR6EvCj0+tDwKU1rpkFrFZK3Q/EA1f5pDohhKiD8/DErJw8/rpsq9vr7F0sWTl5LNn4g8f7WUxQ4+eBIXEWE8nWk9xyyy0sX77c5VxsbCz3338/U6ZMoVWrVt7f3AtGAt1dx07NdWdGAa9orRcqpS4D/qWU6qW1dvlolFLjgfEAHTt2bEi9QgjhVl2jW+z97Quyd3tcNAsaFuYAl57eSM+eN1BRUX0Ds9nMXXfdxSOPPEL79u0bdmMvGXkoegjo4PS6PbW7VO4G3gTQWm8AYoHWNW+ktV6ktU7VWqe2aeO/pSyFEJGnrged9mn/vn4YqoA7+3Vk7PBrXML81ltvZefOnSxatKjJwhyMtdA3Ad2UUp2BPCofet5e45ofgEHAK0qp86kMdP+tVSmEEDV4mslptZhYkL2bh5ZtxVQ1gagxKkqKUGYLyhzFk1X95gBDhw6luLiYRx99lIsvvrhR79FQ9bbQtdZlwAQgG9hF5WiWXKXUHKXUsKrLJgHjlFLbgDeAP2rdyE9NCCG84HbUi0lRVqEdo14aE+a6vJSTX68ib9E4Tu34NzarxWXS0NKlS8nOzg5YmAOoQOVuamqq3rx5c0DeWwgRnpzXYbHFWcg/XVpnn7kRWldQuPMzCr5YTFnBLwCYm7VkSfZXjLz83MYX7SWl1Batdaq7czJTVAgRFmqGeUEjw1xrTdGBzeR/9iqlRw66nLPFx5AcX9ioev1BAl0IERK82S3I3SbR3jhzKJf8z16l+NBOl+Om2OYsmD+L++67j9jY2Ea9hz9IoAshgp67lRUX1zGevKFKjhwk/7NXKdq/yeW4ssTSom8a3a8excSJwzx8d+BJoAshgl5TLKYFcPrbda5hboqiecoQEi67lWa21ky7sbffa2gMCXQhRNDz92Jadi0uuYmTX79PxZlTxPf8HQkD7iDadrbPdj3yNwl0IUTQs8VZGt0v7qyiuJAT/3mX+F6/x5LYznHcFBNPq+seJMp2NtFtOpFks7J+6u999r7+JoEuhAgKnrZ+y8rJ89liWrqshJNfr6Jgw1tUnDlJ6fGfaDNssss1cd36AWBSNHpj6aYm49CFEAFX86EnVE4KsphVrdUTG0JXlHNqxycUrH+d8pP/dTl3zl3PEt2mU63vsVpM7Jo7pNHv7WsyDl0IEdTcPfQsrdCUVjSuwam15vSeL8n//F+UHTvkci4q4SwSrrgTS2v3CwUW+eAHSVOTQBdCBFRWTp7bNVgaq+jgVvI/f5WSn/a6HDfF2bD1v41mFw5Gmf230UUgSKALIQIiKyeP6e9s90mXSk3HPnmRk5tXuBxT0XEkXHozzVOHYYp2v32dM1Pjd4RrchLoQogmY3/w6Y8WuTNr177VgW620OKiG2hx2QjM1hZ1f6OTRvb2BIQEuhCiSbh78OkLZSePYo63oUzVKy1aO/UhtlMKUS3akNB/FFEtvN9/IcnDJtTBTAJdCNEkZr+X69MwLy86yYmNb3Hy61W0HDyBZr1cx4u3HTHLJeS9YbWYQ27IIkigCyH8LCsnj9nv5fpsYlBFyRlObllJwVfL0cWVKx7mr1tCfPIVqKjqh5wNDfOkEJkV6o4EuhDCb+54cQPr9x/zyb10eSmntq0m/8s3qCjMdzlntjanvPA4UQltG3Rvq8VMxvDeIRniziTQhRB+cen8f/PLyZJG30frCk7v+pz8LxZTlv+zy7molknYrvgDcd37o5R3w1JsVgsFRaUhs06LERLoQgif81WYFx3YwvHPXqH01+9cjpubtSJhwO00631Vg7pWEuMs5DxyTaPrCzYS6EIIn7rjxQ0+CXOAooM5LmFuim1Gi3630vyi6zFZYhp0T6vFzMyhPX1SX7CRQBdC+ExWTp7P+swBEvqN4NS2bNAVNE9NI+GSmzDFNmvw/cxKhUVfuScS6EKIRsvKyWPWytwGr4pYVvAL+evfwDbgDpcx4+a4BNoMm0L0WV0xN0tsVI0Wk2LBiAvDNsxBAl0I0UhZOXlMfmtbgxbSKi/Mp2DDMk7mfAgVZShlotWQB1yusXZ1u7CgV6wWExnDLwjrMAcJdCFEI01+ayveLsdSUXyaE/95lxOb3kWXnnEcP7Xj4wbP7HQnlMeUN4QEuhCiQbJy8vift7d5Fea6rISTOR9QsOFNKopOuJyLSTof25VjfBrmobTbkC9IoAshvPZw1g4Wb/zB8PW6opzCb9aQv+51yk8ecTlnaf0bbFeOxtr1Eq/HknsSqlP3G0sCXQjhlaycPK/CHOC/Kx7j9J4vXY6ZE87CNuAO4ntc2eBp+u5EWjeLMwl0IYRXpizf7vX3xPca5Ah0U1wCCZffRvMLr3VZe6WxwmX6fmNIoAshDLv6fz+luKzuTvOSI99jad3RpfvEeu4lxHa+mJikZFqk3ogpJs7Q+yXZrBQWl9U7HDI+2sz8myI7zEECXQhh0MNZO9j7a6HH86XH8sj/YjGnv/2CNmnTiet+ueOcUqpyOVsv+8jt/eATl22lrh8jtrhor+4briTQhRAO9h2FDucXuSxaVVe/ednJoxR8+Qantq0GXRm7xz9/DWu3S136xhvywNPe4q5v+d28/CKmvbPD5XsikQS6EAKovaNQXn4Rk9/a5jFMy8+c4sTGtzm55T10WbHLOUvrDlQUF3q15VtNiXHV/ev5BtZSLyotZ0H2bgl0IYRYkL271o5CpRW6VphXlJ7h5Jb3OLHxbSqKXbtgYjpeQOKVY4hp17ghgxazcllAq53Namgf0sN+3qs02EmgCyEAY2F4cls2BeuWUH7KdQGu6LO6YrtyDLGdUho9ltzdNP3Jg7sb2o+0XQjuA+pLEuhCCMBYK7jk570uYR6V2K5yg4nk/ihl8kkdJWWah5ZtZUH2bkcfvj3c/7psq8fvi9TJRM588ycghAhpWTl5HC8srve6hMtHoaJiMDdrScvBE2h39/PEn3+Fz8IcoFxrNNUPOrNy8oDKh51JHlrg4b4srlHSQhciQtlHtOTlF6EA57USi/N2kf/lUlpf9xDmeJvjeFTzVrQdMYvoc7phssT6vcaaDzrddb3IhKJqhgJdKXUt8DRgBl7SWme6ueZWYBaVfy+2aa1v92GdQggfqjmixR7mJUe+J/+Lf1G0dyMABRuW0fKqe1y+N7Zjb5/VYbWY6+0Xd+7bt4e2u6GV9fE0JDOc1BvoSikz8BxwNXAI2KSUWqm13ul0TTdgGtBfa31cKdWwrbeFEE2i5oiWsoJfyV+3hMJv1uDcVj+1fTW2AXc0apegumQM713vGPOaDzqd+9SNcjckMxzHrRvp+LoE2Ke1PqC1LgGWAjfWuGYc8JzW+jiA1vpX35YphPAle6u3/HQBxz5eRN6L4yn85hOcwzzu/Cs550/P+C3MoTJM46I9tyt99aDT3ZBMe3dOODHS5ZIE/Oj0+hBwaY1rzgNQSq2nsltmltb6o5o3UkqNB8YDdOzYsSH1CiF84Cyr5tt/v1G5wUSJ68iW2M4Xk3jlaKLP6urXGpJsVrJy8uocWeOrvnFPQzLDbdy6kUB3N6i05l5TUUA34HdAe+ALpVQvrXW+yzdpvQhYBJCamur9flVCCJ/If2cWBdu3uByLbtedxCvHENvxAr+/v9ViZmByG0e3hztJNqvPukM8DckMt3HrRrpcDgEdnF63Bw67uWaF1rpUa/0dsJvKgBdCBFBWTh79M9fQeer79M9c4xgCOGva3xzXWFp1pO3whzn7zieaJMztQwzXfnukzgeihcVljnoba/Lg7lgtrmuuh+O4dSMt9E1AN6VUZyAPuA2oOYIlCxgFvKKUak1lF8wBXxYqhPBOVk4eU5dvJ//AVmI69HZ5EBjdrT8te1yOqdOlJPW9mvwzXm4K2kDOQwwfqmOSEEB+UanPHlw2ZnRMKFFa19/zoZS6DniKyv7xf2qt5yul5gCbtdYrVeVc34XAtUA5MF9rvbSue6ampurNmzc3+j9ACOFer/FPsvf9v1NyeDdtR8zG2uViAGxWC8VlFfUOF/QFi1kRHx1FQVFprRDtn7nG0Poskbg3aF2UUlu01qnuzhkah661/gD4oMaxR5y+1sDEqv8JIQIoJyeH6dOnk/tR9biE45+9SmznFJQy1btZhC+VlmviY6KYNaxnrdaw0fVZwu3BpT/J1H8hwsS+ffsYNWoUF110ER85hTnmKGI79kaX+SfIzfUsxlVzCr9dWkoSGcN7k2Szouq4T7g9uPQnmfovRIj76aefmDt3Li+++CJlZWWO4yaTiea9BtHs8lFEJfhnrp/VYubmi5NYviWvzpa2p7XKnScJ1Zz8Y79/uD249CdpoQsRorTWPPzww3Tt2pUXXnjBJczT0tLYvn07E2YvJMZ2ll/eP8lmJWN4b+al9XZpaXtSX9dJzRa7/f7h9uDSn6SFLkSIUkrxww8/UFRUHZRXXnklmZmZ9OvXj6ycPJZv2UG5gYEP3nC3GJZzS9vTw04jXScNmdYvqkkLXYgQNnv2bCwWCykpKXz00UesXbuWfv36Ae6nuzeWzWqpt9UcKWO+g5G00IUIchUVFbz99tssXLiQjz76iMTERMe5zp07s3HjRvr06YPJ5No+8+XokMQ4CzOH1h6p4k6kjPkORhLoQgQprTX//ve/mTZtGl9//TUAjz/+OBkZGS7XXXTRRW6/3+g+nEbkPHKNV9dL10lgSJeLEEHoq6++YtCgQQwePNgR5gCvvfYaxcX17ywE7rs+RHiTQBciiOzatYvhw4fTr18/1q5d6zgeGxvLlClT2LFjBzExMYbuZR81IiKHBLoQQeCHH37grrvuolevXrz77ruO42azmXvuuYd9+/aRmZlJy5YtvbpvXftwGv3H/9TIPl69pwgcCXQhAqyiooJBgwbx8ssvU1FRvUjWyJEj2bVrF//3f/9HUlLD+6M9jToxshzXnf06Sl94CJFAFyLATCYTU6dOdbwePHgwW7ZsYenSpXTr1vhVqD1N2KlPlEkxL026bEKJjHIRogkVFxfz4YcfkpaW5nJ8zJgxrFmzhrFjxzJw4ECfv6+7USd/rWf52mYxEg+hRlroQjSB8vJy/vWvf5GcnMxNN93E559/7nI+KiqKJUuW+CXMPenfte7++IImXJVR+IYEuhB+pLXmvffeo0+fPowePZqDBw8CMHXqVIzsReBPI1I7Yqpj8RVZ5TD0yO9UQvjJF198wdSpU/nyyy9djrdq1YoRI0ZQUVGB2Ry4ceILsndT4eFnikzVD00S6EL42LZt25g+fToffOCyJwzx8fFMmjSJSZMm0aJFiwBVV62upQFklcPQJIEuhI+UlZUxZswYXn/9dZfjFouFe++9l/T0dNq29c+65A3haWmA+jasEMFL+tCF8JGoqCiXaflKKcaMGcOePXt4+umngyrMwfPSAOVau91hSAQ/CXQhGsh5EpDdvHnzMJvNDBs2jO3bt/PKK6/QqVOnpi/OAPv4dHctcvsOQyK0SKAL4aWioiKeeOIJevXqxalTp1zOJScns2fPHlasWEGvXr0CVKFxaSlJVHgYbSObM4ceCXQhDCorK+Oll16iW7duTJ48mV27dvHUU0/Vuq5Lly4BqK7hPA1PlGGLoUcCXYh6aK15++236dmzJ+PGjSMvr7pv+Z133nHb9RJKZIeh8CGBLkQdPv74Y/r27cuIESPYs2eP4/jZZ5/N888/z8aNG2vtFBRqZHPm8CHDFoVw4z//+Q/Tpk1jzZo1LscTEhKYMmUKDzzwAPHx8QGqzvdkh6HwIIEuRA3FxcUMGzaMX375xXEsNjaWBx54gClTpni9JrkQTSW0f1cUwg9iYmJIT08HKjeYGDduHHv37uWxxx6TMBdBTVroIqIdPXqUVatWMWbMGJfj48ePZ9euXTz44IN07y4PB0VokEAXEenUqVM89dRTLFiwgBMnTtCjRw/69u3rOB8TE8Pzzz8fwAqF8J50uYiIUlJSwrPPPkvXrl2ZMWMGJ06cAGDatGkBrkyIxpMWuogIFRUVvPHGG8yYMYPvvvvO5VxycjL33XcfWmuULEwlQpgEughrWms++OADpk+fzvbt213OtW/fntmzZzN69GiiouSfggh98rdYhK3CwkKuvfZa1q1b53K8ZcuWpKenc9999xEbGxug6oTwPQl0Ebbi4+Np1qyZy+uJEycyadIkEhISAliZEP4hD0VF2HBei9wuIyOD6OhoJkyYwP79+5kzZ46EuQhb0kIXIe/nn39m3rx5fPjhh+Tm5rp0o/Tp04e8vDxat24dwAqFaBqGWuhKqWuVUruVUvuUUlPruO4WpZRWSqX6rkQh3CsoKODhhx+ma9euPPfccxw4cMDt2HEJcxEp6g10pZQZeA4YAvQARimleri5rjnwAPCVr4sUwllRURELFy6kS5cuzJ8/n9OnTzvOff755wGsTIjAMtJCvwTYp7U+oLUuAZYCN7q5bi7wOHDGh/UJ4VBWVsY//vEPzjvvPP72t79x7Ngxx7nevXuzatUq3n333QBWKERgGQn0JOBHp9eHqo45KKVSgA5a61V13UgpNV4ptVkptfnIkSNeFysik9aa5cuX06tXL8aOHcuhQ4cc5zp37szixYvZunUr119/vUwMEhHNyENRd/9CHJsQKqVMwJPAH+u7kdZ6EbAIIDU11f1GhkLUUFBQwNixY8nPz3cca9u2LY888gjjxo0jOjo6gNUJETyMtNAPAR2cXrcHDju9bg70Aj5VSh0E+gEr5cGo8BWbzcbUqZXP4lu0aMG8efPYv38/f/nLXyTMhXBipIW+CeimlOoM5AG3AbfbT2qtCwDHMAKl1KfA37TWm31bqogEu3fv5pNPPuG+++5zOX7//fdTWFjIgw8+SKtWrQJUnRDBrd4Wuta6DJgAZAO7gDe11rlKqTlKqWH+LlBEhkOHDjF+/Hh69uzJhAkT+Oabb1zOx8XFMWfOHAlzIeqgtA5MV3ZqaqrevFka8ZHu2LFjZGZm8swzz3DmTPUAqaFDh7Jy5coAViZEcFJKbdFau+3SlpmiIiAKCwt5+umnefzxxykoKHA5N2jQIGbMmBGgyoQIXRLookmVlJTw0ksvMWfOHJdNmAFSU1PJyMjgqquuClB1QoQ2CXTRZH799Vcuu+wyDhw44HK8e/fuzJ8/n+HDh8s4ciEaQVZbFE2mTZs2dOhQPQI2KSmJl156iW+++Yabb75ZwlyIRpJAF35Ts29cKUVGRgYtW7bkiSeeYO/evdx9992yW5AQPiL/koTPffPNN6Snp7Nz50527tyJxWJxnLvsssv48ccfiYuLC2CFQoQnaaELnzl48CBjxozhggsuYOXKlezbt49//OMfta6TMBfCP6SFLhrt119/Zf78+bzwwguUlpY6jiul2LNnTwArEyKySKCLBjtx4gQLFy5k4cKFFBYWupy7/vrrmT9/PhdeeGGAqhMi8kigC6+dOXOGF154gfnz53P06FGXc5dffjmZmZlcccUVAapOiMglgS68duLECWbMmOHSKu/VqxePPvooN9xwgww/FCJA5KGo8Frbtm2ZNGkSAJ06deK1115j69atDB06VMJciACSFrqo09q1a8nJyWHixIkuxydNmkTbtm0ZO3YsMTExAapOCOFMVlsUbm3ZsoXp06ezevVqzGYzu3btolu3boEuS4iIV9dqi9LlIlzs2bOHkSNHkpqayurVqwEoLy9n9uzZAa5MCFEfCXQBwOHDh/nzn/9Mjx49ePPNNx3HTSYTd911F48++mgAqxNCGCF96BHu+PHjPPbYYzz99NMuG0wADB8+nHnz5nH++ecHqDohhDck0CPY3r17ueSSS8jPz3c5PnDgQDIyMrj00ksDVJkQoiGkyyWCnXvuuZx77rmO1xdddBGrV6/mk08+kTAXIgRJoEeIiooKDh8+7HJMKUVmZibdunVj2bJlbNq0iauvvlrGkgsRoqTLJcxprcnOzmbatGmUlpaybds2zGaz4/ygQYPYuXNnUK1JnpWTx4Ls3RzOL6Kdzcrkwd1JS0kKdFlCBD1poYexDRs2MHDgQIYMGcLWrVvJzc1lyZIlta4LtjCf9s4O8vKL0EBefhHT3tlBVk5eoEsTIugFz79k4TO5ubmkp6ezYsUKl+NWq7XWYlr+0JgW9oLs3RSVlrscKyotZ0H2bmmlC1EPCfQw8v333zNz5kxee+01nGcAR0VFMW7cOGbMmME555xj+H4NCWZ7C9seynn5RTy0bCt/XbaVJAP3OJxf5NVxIUQ16XIJA0ePHuWhhx7ivPPO49VXX3UJ81GjRrFr1y6ef/55r8Pc266PrJw8Jr25rVYL216NPdwfztrh8R7tbFavjgshqkmgh4HTp0/zwgsvUFJS4jg2ZMgQvv76a15//XWXoYlG1dX14Y79B0B5PWsDaWDJxh88/mCYPLg7VovZ5ZjVYmby4O7GixciQkmgh4EOHTowYcIEoHIT5k8//ZQPPviAlJSUBt/TUxdHXn4R/TPXuASyp5a5JxqY/V4u/TPX0Hnq+y73S0tJImN4b5JsVhSQZLOSMby39J8LYYCsthhCysvLWbx4Mb/88gv/8z//43Lu6NGjrFu3jmHDhnk1jtxTP3n/zDXk1dFvbbWYyRjeG8Clz9wXzEox6tIOzEvrbbheISJFXastSqCHAK01K1euJD09ndzcXGJiYtizZw8dO3asda03gVfzASZ4F9RmpWhhjeL46VKP1zTGnf06uoR6XfVKqItIIYEewj777DOmTp3Kxo0bXY5PmDCBZ555xuWYp8C7+eIk1n57xKtWeJLNysDkNqza9hP5Rf4J7PqYFJyTYHXUfbqkzO0PjySblfVTfx+ACoVoehLoISgnJ4fp06fz0UcfuRxv1qwZkydP5qGHHqJ58+Yu5+rrJrGzt2ofWraVwPzp+16SzSrdMCIiSKCHkH379jFjxgyWLl3qcjw6OprBI8bwc6drOVIW4wguwNHFEi7h7C0FLv/t0g0jwlldgS4Ti4LIV199xYABAygrK3McM5lMjBkzhvh+I1l1oAxddSovv4jJb20DBaXlkRrltcMcZGapiFwybDGIpKamkpyc7HidlpbGjh07GHb/3Mowr3F9aYUO+TCPNiuSakwaMitV519Mm9XiGNLo6b9eZpaKSCQt9AA5ffo0P//8M126dHEcM5vNPProoyxcuJDMzEx+junAuA0wryYAABA4SURBVJW7ycv/LoCV+pfWuO3z7jzt/dpN7yrFZRU8ObJPnQ92ZWapiEQS6E2stLSUl19+mdmzZ2NNaE3bP/wvhwvOOLoOtFYkDpzGMzs0X+4Pn4eWnpRWaLfdI3U92nHuUpk8uLvbkT0ys1REIkNdLkqpa5VSu5VS+5RSU92cn6iU2qmU2q6U+kQp9RvflxraKioqePPNN+nZsyf33HMPhw8fZv+u7ez96mOgujGqlCK/qIz1+4+FfZjb1WxhG1kq196lIjNLhahWbwtdKWUGngOuBg4Bm5RSK7XWO50uywFStdanlVL3Ao8DI/1RcKjRWrN69WqmT5/O119/7XLOHJ8YoKqCz8NZO0j9TUsWZO82NPTSuUslLSVJAlwIDAxbVEpdBszSWg+uej0NQGud4eH6FOBZrXX/uu4bCcMWN27cyLRp0/j0009djttsNrjwRppfPBSTJTYwxYUwe/eUzWpBKcg/XSrjz0XEaOywxSTgR6fXh4C6dhC+G/jQQyHjgfGA22nroc4+7f77g99R9MXLHNu53uV8bGwsDz74IFOmTOGGv+cYaomK2uxNEOcZrPblfQEJdRGxjPShu1vpyW2zXil1J5AKLHB3Xmu9SGudqrVObdOmjfEqQ4DL+uG6gmPfVk/VN5vN3HPPPezbt4/MzEwSExPdLhMrGqeu5X2FiARGWuiHgA5Or9sDh2tepJS6CkgHrtRaF/umvODkbgEs5/XDLYntaHbhYE7lfECrC37HhrcX0a1bN5d72FuR9j5jdxNkhPdk/LmIZEYCfRPQTSnVGcgDbgNud76gqt/878C1WutffV5lkMjKyWPWylzHr/oVxafJff8N7vrIQot+t7hca7t8FM0uuIbYs891hHnNHwQDkyt/S1FAlAlKK5r0PycseTP+XJbiFeGm3kDXWpcppSYA2YAZ+KfWOlcpNQfYrLVeSWUXSzPgraq1uH/QWg/zY91NznklQ11WysmtH1KwYRkVpwtQlhjiew9yGbVibpaIuVmiI2Dc7bW5eOMPjuslzH2jrvHnzgFui7Nw6kwZpRWVvxdJH7wIB4YmFmmtPwA+qHHsEaevr/JxXUFnQfZuTheXULjzM/LXLaG84BfHOV1azMmcD7ENuL3W99lb4e62dBO+FWcx1Qpje4jX7NZytwyvrAEjQp3MFK2DIwyOn+b0/v+Q/9lrlP73e5drzM3bYLviDuJ7DnR7jze++pHU37SUvt0mcLq0gv6Zaxyt9Nnv5boEt5FnFPLnJEKZBLoH9i6S4we2kf/ZqxTn7XI5b7K2IOGykTRPGYKKivZ4n3Kt+euyrW6HCgnfa+wqlLIGjAhlEugeLMjezdFdGziyfI7LcRVtpUXfNFr0vQlTTJzh+8kIlqZj7xf3lsWkZA0YEdIk0D04nF+EtfNFRCWeQ9nxn8AURfOU60i47FbM8bZAlyf8QX6NEiFOAr3KP1d/zbPZOzhuaU07m5W4aDOFJWC7YjRFBzZhG3AHUQlnBbpM4Uel5e5XfhQiVET8BhdLPt/J2b8dxdgbLmfnmwuo0Jq8/CIKSypHpMSffwWtr58oYR4h5KGoCGURG+hFRUWM+evDjB58Kb98sRRdWkzxoVzOHNgS6NJEAMlDURHKIi7Qy8rKePHFF+nWrRuvPT2fijOnHOcsbbugomX1w3CUGGfBaqn/r7t93oAQoShi+tC11ixfvpz09HT27Nnjci7Kdg62K+4k7vwrUCrifsZFhIKiUowMfln77RH/FyOEn0REoG/ZsoV77rmHLVtcu1PM8Ykk9K9cc0WZI+KjiFhGRzJKH7oIZRGRYrGxsS67BamYeBIuvZnmFw/DJF0swon0oYtQFhGB3rNnT0aPHs2yZctIGTKKHztcg9naPNBliSAjm0uLUBdWHcY//vgjY8eOZcGC2vtrZGRk8GzWFxw+72YJc1GLbC4twkFYtNCPHj1aGdjPPktxcTE2m42xY8eSmFi9nO0555zDP1/dVcddRCRbP/X3gS5BiEYL6Rb6qVOnmDdvHl26dGHhwoUUF1dulJSfn89bb71V63p54CXcSYyzBLoEIXwiJFvoJSUlLFq0iLlz5/Lrr64bJPXt25eMjAxOtkymf+Yal91o2tmssjFzBFMKdI3RLhazYubQnoEpSAgfC6kWenl5OYsXLyY5OZn777/fJcyTk5NZvnw5X331FSdbJldv2Ez1bjQDk9tgMckKTBFLw1Mj+5Bks6Ko7DdfcMuF0m8uwkZItdBfeeUVxo4d63Ksffv2zJ49m9GjRxMVFUVWTh6T3txGeY2mWFFpOau2/cSCERe67AsqwofVYq5zV6h2NitpKUkS4CJshVQL/Y477qBDhw4AtGrVioULF7J3717uuusuR5hPe2dHrTC3s4f41pnXyEqpYcY+SiXJwzhyRd37jQoRDoK2hb59+3aio6NJTk52HIuNjSUzM5Nvv/2WSZMmkZCQ4PI9RvbttC+PKv3p4cFqMdcabui8GTdUhvkd/TpKy1yEvaAL9AMHDjBz5kyWLFnC4MGD+fDDD13O33577Y2Y7YyMYrFf06lVZAe6xawatEVbMLFaTLXC3P71guzdLg/EJcxFJFDaQ/eEv6WmpurNmzc7Xv/yyy/MmzePv//975SWVvdvr127lt/97neG7pkyZ7Xb3dydxUebiY4y1XtdOEuMszBzaM/KDbBD6IeaWSnKtcasFKMu7cC8tN6BLkmIJqeU2qK1TnV3LuAt9IKCAp544gmefPJJCgsLXc7dcMMNnHWWsY0lsnLyOHWmrN7rCkvKHZtXRCKrxczMoT1JS0li8/fHWLzxh0CXZEiSzSqTf4SoR8AeimqtWbhwIV26dGHevHkuYT5gwADWrVvHe++9x/nnn2/ofrPfy23w5sCRpKi0nAXZuwFYte2nAFdjjKyxIoQxAetyiY6O1s5dKwC9e/cmIyOD6667DqWMj0PJysnjr8u2+rrEsJYYZwnKbierxczNFyex9tsj0gcuhBtB2eXiHOadO3dm7ty5jBo1CpPJ+18a7C1OYZwvw1wBvmgW2Pv2JbyFaJiABXpiYiLR0dHMmDGDcePGER0d7fU9snLyQu7BXlNQwOVdW5J7+GSDJlCZTYpyg91Xzi3quv4caoa+xayIj46ioKhUWuFC+EjAAr1Dhw6sX7+eZs2aNej77ZOI6hp37quWY6B4U3+cxURRaUWtcPQ0c9YTk4KFIy4k/d0d9T48TnITxO7+XKQbRYimEbBAt1gsDQ5zqH8SUX3TwL1hMStG9u3gaIXah8811lMj+9T5Q0mD4ffSKJ4c2adWSKalJPGQwecLNSfpTH57m8ex6p5Gncg4cCECJ+DDFhuqrklE9pajL7pj3LVC7fpnrnF7f/vMxPe3/+Sxrzqpal0RwGML2v7e9f0mAtWjV9zVaWRWrM1qYdaw6v5r+/+7W/emvlEnsl6KEIERUmu5OPO096O95ZiWksTkwd2xWswNur/ZpHhqZB/Hvdxxd397mM9L603OI9dwZ7+OtdaNcQ7EtJQkFt56Ya372K9JS0lyWaOkrrE/nn7IeaoTKj+vp0b2YevMa9y27rfOvKbWCoWys48QwSloZop6y1Nfbc2wqW9I48HM68nKyXNpiXoz2sL+YLau7gVfXWO/rq4WvafJN0bvL4QIbnUNWwzZQAfjIeWpayRUZx8a/WEmhAg/QTkO3ReM9tW664cO5dmH8uBRCOFOSAe6UeEYgPLgUQhRU0QEOkgACiHCn6FRLkqpa5VSu5VS+5RSU92cj1FKLas6/5VSqpOvCxVCCFG3egNdKWUGngOGAD2AUUqpHjUuuxs4rrU+F3gSeMzXhQohhKibkRb6JcA+rfUBrXUJsBS4scY1NwKvVn39NjBIebNcohBCiEYzEuhJwI9Orw9VHXN7jda6DCgAWtW8kVJqvFJqs1Jq85EjRxpWsRBCCLeMBLq7lnbNwetGrkFrvUhrnaq1Tm3Tpo2R+oQQQhhkJNAPAR2cXrcHDnu6RikVBSQAx3xRoBBCCGOMBPomoJtSqrNSKhq4DVhZ45qVwJiqr28B1uhATUEVQogIVe84dK11mVJqApANmIF/aq1zlVJzgM1a65XAP4B/KaX2Udkyv82fRQshhKgtYGu5KKWOAN/74Fatgf/64D7+JnX6XqjUKnX6VqTX+RuttduHkAELdF9RSm32tFBNMJE6fS9UapU6fUvq9Cxk10MXQgjhSgJdCCHCRDgE+qJAF2CQ1Ol7oVKr1OlbUqcHId+HLoQQolI4tNCFEEIggS6EEGEjZALdwJrsv1VKfa2UKlNK3RKIGqvqqK/OiUqpnUqp7UqpT5RSvwnSOv+slNqhlNqqlFrnZsnkoKjT6bpblFJaKRWQ4WwGPs8/KqWOVH2eW5VSYwNRZ1Ut9X6mSqlbq/6e5iqlXm/qGqtqqO8zfdLp89yjlMoP0jo7KqXWKqVyqv7dX+e3YrTWQf8/Kmeo7ge6ANHANqBHjWs6ARcArwG3BHGdA4G4qq/vBZYFaZ0tnL4eBnwUjHVWXdcc+BzYCKQGY53AH4Fnm7q2BtbaDcgBEqtetw3GOmtcfz+Vs9iDrk4qH47eW/V1D+Cgv+oJlRZ6vWuya60Paq23AxWBKLCKkTrXaq1PV73cSOViZ03NSJ0nnF7G42b1zCZgZC1+gLnA48CZpizOidE6g4GRWscBz2mtjwNorX9t4hrB+890FPBGk1TmykidGmhR9XUCtRc39JlQCXQja7IHA2/rvBv40K8VuWeoTqXUX5RS+6kMyweaqDZn9daplEoBOmitVzVlYTUY/XO/uepX7reVUh3cnG8KRmo9DzhPKbVeKbVRKXVtk1VXzfC/papuy87AmiaoqyYjdc4C7lRKHQI+oPK3Cb8IlUA3tN56EDBcp1LqTiAVWODXitwzun79c1rrrsAU4GG/V1VbnXUqpUxUbnk4qckqcs/I5/ke0ElrfQHwMdU7fDU1I7VGUdnt8jsqW74vKaVsfq6rJm/+zd8GvK21LvdjPZ4YqXMU8IrWuj1wHZULGfole0Ml0I2syR4MDNWplLoKSAeGaa2Lm6g2Z95+nkuBNL9W5F59dTYHegGfKqUOAv2AlQF4MFrv56m1Pur0Z/0icHET1VaT0f0NVmitS7XW3wG7qQz4puTN39HbCEx3Cxir827gTQCt9QYglsqFu3yvqR8iNPDBQxRwgMpfq+wPHnp6uPYVAvdQtN46gRQqH6J0C+bP07k+YCiVSyUHXZ01rv+UwDwUNfJ5nuP09U3AxiD+s78WeLXq69ZUdim0CrY6q67rDhykapJkkH6eHwJ/rPr6fCoD3y/1NvkH0IgP7jpgT1UYplcdm0NlKxegL5U/LQuBo0BukNb5MfALsLXqfyuDtM6ngdyqGtfWFaSBrLPGtQEJdIOfZ0bV57mt6vNMDkSdBmtVwP8CO4EdwG3BWGfV61lAZqA+S4OfZw9gfdWf/VbgGn/VIlP/hRAiTIRKH7oQQoh6SKALIUSYkEAXQogwIYEuhBBhQgJdCCHChAS6EEKECQl0IYQIE/8PAVF9GVhxzhgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(preds_df3[(preds_df3['actual']==1) & (preds_df3['t1']!=preds_df3['t2'])]['t1_attn'], preds_df3[(preds_df3['actual']==1) & (preds_df3['t1']!=preds_df3['t2'])]['t1_cont'])\n",
    "xpoints = ypoints = plt.xlim()\n",
    "plt.plot(xpoints, ypoints, linestyle='--', color='k', lw=3, scalex=False, scaley=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 382,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1a8f6a0450>]"
      ]
     },
     "execution_count": 382,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deXyU5b338c+VyYRMQshEEheCLMomikIJikQOhRZBcUGxRRE81oXqUfuAHI6ILAnQikIrxUcfD7V4WkEWLaZYUTwVXEBQQHY0CESQRAsqA4EkZJJczx+ZGeaezJZk9vzer5evV+a+78z9Y8Bvrlz3tSitNUIIIeJfUrQLEEIIERoS6EIIkSAk0IUQIkFIoAshRIKQQBdCiASRHK0bZ2dn606dOkXr9kIIEZe2bdv2vdY6x9u5qAV6p06d2Lp1a7RuL4QQcUkpddjXuYBdLkqpxUqpY0qpPT7OK6XUQqXUAaXULqXUT5pTrBBCiKYJpg/9f4Dhfs7fAHR1/Dce+H/NL0sIIURjBQx0rfVHwI9+LrkV+KuutxmwKqUuClWBQgghghOKUS65wDdur486jjWglBqvlNqqlNp6/PjxENxaCCGEUygCXXk55nWBGK31Iq11ntY6LyfH60NaIYQQTRSKQD8KXOz2uj1QFoL3FUII0QihCPTVwD2O0S79gZNa629D8L5CCCEaIeA4dKXUMuCnQLZS6igwEzADaK1fAtYANwIHgArgV+EqVgghhG8BA11rfVeA8xp4JGQVCSFEHCjaXsq8tcWU2SppZ7UweVh3RvbxOh4kYqI2U1QI0bLEYgA2VdH2Up5ctZtKey0ApbZKnly1GyCqfyZZnEsIEXbOACy1VaI5F4BF20ujXVqTzFtb7Apzp0p7LfPWFkeponrSQhdChJ2vAJy0cicQ3VatL0XbSyl8ay8nKuwAWC1mCm65nJF9cimzVXr9Hl/HI0VFa0/RvLw8LYtzCRE97l0g1jQzWsPJSntYukM6T3nb++QU6ieyaCC3ifcNR1dO0fZSJr2+k9o671WblKLWS3bmWi1snDKkWfcORCm1TWud5+2ctNCFaIE8+4CdrVDw3h/c3NBsZ7VQ6qP16oxFX/3Q/u7trS97woodFL61l5k3X97kYC98a6/PMAe8hrkCBvcwTpiM9HMDaaEL0QLlz13nM2CdnK1Nz9AEsJhNPH17r6DDydt7BLqvr+9zv3egP0dWmrlBsLuHbKbFjFJgq6j/zWRwjxzWf3k84Gfji3ttofjcvJEWuhAJprktv2D6ep2hFugBoL86PMPzbE0tfhq+DWrzde8JK3ZQsHovtkq757cbnKiwM2HFDrYe/pE5I3s1CFn37y+1VbJk8xH/xQXg/FxG9sn1+7k1NdADNcAl0IWIM6EYMuevC8RJOe7lK/yd93WvY+KKHUxYsYOsNDNV9loq7XWu622V9qCG1bWzWlxf+/vBEyjM3S3ZfIS8jud5DdlQK7VV+v3cmvLg9PPPP2f69Olcd911fq+TLhchYpxna7yiusbQ5+3k7F4IpuU+rWh3UK3RXEe4egt/Xw8GQyXXauHM2ZpGBXeg9ytzDJuMBKXA28fTmAene/bsYebMmaxatQqArKwsTpw44bPLRQJdiBjWmL5nb3yNIAmmD93b+zhZzKawt3QBzCYFGuyB+mniRLB96Pv376egoIDly5cbulmUUmitfQa6TCwSIoY1t4vAcwRJ0fZSiraXNvqhn+bcOtm5VgtP397L1XoPJ3utpiZKjc5QUY7/nJ+bvzAvKSnhV7/6FZdddhnLli0zhPmoUaPYvXu333tJH7oQMcjZzdLU0RbeVNprKXxrL1Vu/dqN4WzpO0e+VFTXhKw2v/eN7zxHAwtG9/Yb5MePH2fGjBm8/PLL1NQYP9ebbrqJWbNm0adPn4D3kkAXIgo8+8UH98jh7V3feu0bD6Xmvn+Z44Ffc7qBEklWmpnqmjrOVPv/LAKNbElKSmLp0qWGMB86dCizZs2if//+QdcjXS5CRFDR9lJ6F77HhBU7DOuaLNl8JOxhHgrtrJaIjBSJF2kpyVQECHMIPLKlbdu2PP744wAMHDiQDz/8kPfee69RYQ7SQhciYuKpZWtOUtRBg9mSvkbYtFTO37ACdY05h2LabDZ+//vfc/LkSRYuXGi4ZuLEieTn5/Pzn/8cpbzt7BmYtNCFiJBwtWzD8XDSXqe9Tn2XMDdyDg21mE0+r7GYTTx6XS5z5syhc+fOzJkzhxdeeIH9+/cbrsvMzGTo0KFNDnOQQBciYkL5gNPd5GHdw/K+wj+L2eQaCuoc9aOo71e3Wswo4MI0xTWnN/LYyHymT5+OzWYDoK6ujj/96U8hr0m6XISIgGlF/oebNceEFTvC9t6innOSkHMylee4/pF9cg0PPc+ePcuiRYv43e9+x6fffWd4r65du1JQUMDo0aNDXqe00IUIgaLtpeTPXUfnKW+TP3ddg40bln36TZQqiz8Ws4mx/TvQjJ6HkHMOnXTOjP32ZCVbD//Y4Dq73c6iRYvo0qULv/nNb/jOLcw7derE4sWL2bdvH2PGjMFk8t1N01QyU1SIZvL2sNNsUqSnJHOy0k6mxRyy6euJypndziGcf9tWGhcPj8f278Cckb1cr9euXcvw4cMN1+Tm5jJt2jTuu+8+UlJSmn1Pf6stSgtdiGby9rDTXquxVdrRNG4RqZZKA8+N7s3GKUP4x85vIx7mWWnmJn3f0k+N6+Fcf/31rqGG559/PgsWLODAgQM89NBDIQnzQKQPXYhmiva2Y4miYPVepq7aRUUTZ7I2VZZjt6bG0FpT+dVmVHIKMMJ1XCnF3Llz+fTTT3nkkUdIT08PbbEBSKAL0UzBjEMWgUXjNxmzSXG6qiboxb+01lQd2oZtwxKqvztA8nntqamZSnLyuSgdNGgQgwYNClfJfkmXixDNJMMG45NJ1T/nCDbMKw/v5F9LJnPsjQKqvzsAQM2PR3nttdfCWWajSKAL0Uwj++Q2uQ9WRM/vf3kVJ4P4raDq6F6+W/Ykx5Y/xdmyL13HVXIKt477NTfccEM4y2wU6XIRohHcV0F0jkm2WsxU1/h/iJfkGMecaTFTfrbG7wbEInL8dZed/XY/to+XUFXyufFEUjIZvYez4OkC7ru+bwSqDJ600IUIknN4ojMAnGOSbZX2gA/y6nT9KI70VslewzwphsZctxRPrtrN4B459ZtoeCjf8S7f/fVxY5irJFpfNYzcXy+i1y8mxlyYg7TQhQhac9di8bcwlzTYI6/SXss/dn6Ltz3pLF2uRr2fgq6pBpVEes9BZObfhTmrnWvKfyySQBciSM0dnlhpr/W5z2Q05frZpzTR2Srt2E+UkZRiwZSe5Tqe3Po8Mn5yEzWnjmPNH4M5+2Lg3L6twW7GHWkS6EIE4Ow3D0UOx1qYAwzukRPUhtHhlKQi/1tKzcljnPxkOad3/5OMPjdw3tCHDeetP/0VyUlJ1Gntd8PtWCKBLoQf8bSGeVOkp5j427bSwBeGWSTDvKb8B05tXkn5jrVQV79DUPmOtbS5ehTJmee7rlNKUac1JXNH+HqrmCOBLoQfzek3V3jtno0pgbZOiwdp5iRu79ue9V8ep8yxC5Q3tWdsnPz0DU5vX1PfN+6mVfueaPvZBt/TLgIbYYeSBLoQfjS139xiNjGqby7rvzwus0jDyHNxLID8uesMn3ltZTmnPltF+ba30PYqw7Wtci8jc+BYLB2vavDesfzw0xcJdCH8aMq0fs8HZ54BI0Ij12ppEOZgfCZQvuNdTqxfjK6uMFyTcmEXrAPHkdr5Jw12CFIQN33mniTQhfBj8rDuAfvQnV0rnpseOAUK8yTH98d690ws8dd6Xv/lcdfXSa3SDGFuzumEdeBYLF2u8brVW67VwsYpQ0JfcIQEFehKqeHAHwET8LLWeq7H+Q7AXwCr45opWus1Ia5ViIhzhvO8tcWuDYEH98hx9dcG05Jzzij1JTPNjK0FDhlsKl8/OKuqqmjVqpWhmyytx3WYN61E19Zgve5u0nrko5T3+ZTx2MXiKWCgK6VMwAvAUOAosEUptVprvc/tsmnASq31/1NK9QTWAJ3CUK8QEee5vVhj+QtzAFuFXVZsDILVYqbgloZjwKurq/nzn//MnDlzWLRokeGzVCqJ8++Yian1eZzXOpW0lOQm/2COB8G00K8GDmitDwEopZYDtwLuga6BNo6vM4GyUBYpRDzLDRDWznBZuvlIi+926Xp+OkdPVHnt4rJV2nlyVf3erCP75FJTU8Nf//pXZs2axeHDhwGYPn06019ezdQ397jeI7lNDhazKaYnBIVKMGu55ALuGyIedRxzVwCMVUodpb51/pi3N1JKjVdKbVVKbT1+/Li3S4RIOJOHdcdi9r5/pMVscm251tLDHOCRwV15+vZeWC3eV6+stNfy7Dv7WLJkCZdddhn333+/K8wBvv32W3pn2Xn69l7kWi0o6n+gPn17r4QPcwiuhe5t2SDPf3t3Af+jtf69Uupa4FWl1BVaa8OKRVrrRcAiqN9TtCkFCxFv3Pvh3VdpzHW0zJd9+k3AbhlzksJsUhHfzSfS5q0tZuOUIcxbW9xgwwut66go/oStG5Yy7gfjptvZ2dlMmTKFhx9+mLS0NDpDiwhwT8EE+lHgYrfX7WnYpXI/MBxAa71JKZUKZAPHQlGkEPHOWz+8cxaqvzB3H0IHMHHFjrhvyfub5u98oOk5/r/i4BZsH/0V+7ESw3Gr1crkyZN57LHHyMjICEu98SSYQN8CdFVKdQZKgTuBMR7XHAF+BvyPUuoyIBWQPhXRojnXgPH10C3QLFRvQ+gmrNgRtnojxZSkqKv1nujOmZmeD4krijcawjwjI4OJEycyceJErFZreAuOIwH70LXWNcCjwFrgC+pHs+xVSs1SSt3iuGwS8KBSaiewDLhX61hchkiIyHBfO11TPxb9yVW7Kdp+bt0Uf7NQfQ2hy42zqeje+mvtPsJccW47P8/nDtb8uyApGXOrVJ544glKSkooLCyUMPcQ1AYXWus1WutuWutLtda/dRybobVe7fh6n9Y6X2t9lda6t9b6vXAWLUSs89b6rrTXMm9tseu1v3VCRvX1PlRy8rDuXjdkiFWNadVp4MKz3zBy5EgGdkg1PNjs2LETk+f+X745/DVz586lbdu2Yao4vslMUSHCwFfru8xWadjGzpf1Xx732mUDUOujhRtpoVx87Ox3Bzi7eRnXPvMpAD169GDu3LkeP9TidwZnpKho9Yzk5eXprVu3RuXeQoSbr/VbstLMVNnrglrB0WI2Ga4zmxQ1tTomHopmpZmbvCGG2aRAg71OU338a05ueI2K/Z8YrmnTpg1lZWWkp6eHotyEopTaprXO83ZO9hQVIgy8jT23mE1oTVBhblKqwXX2Joa5ycuaJU5N7bxJS0ludH++c0z4vDuuYsLVGZx59w98u/gxQ5grpRgzZgxbtmyRMG8CCXQhwmBkn1yvk1tOVgZu1VrMpoDj0hvD13tZLWbu7t/B66SnQJtWl9kq/U6Y8pRrtVAydwRLRnfm7wun8ZtRP+X7netw77QZNWoUu3fvZunSpXTr1i2o9xVG0ocuRJh4G3vuq+/c5Ngdx9lXHqiPvSk87+Fem+eyAyalyLQk++xWaWe1+Fy47G/bSg2/XThH7GitGTZsGF999ZXhvW666SZmzZpFnz59QveHbaEk0IWIIG/L8VrMJq9T00O99Z2v7dTWf3m8QVeOvU6TlpLMzJsv91qv8wGttx9aeR3P8zn+fsqUKdx///0ADB06lFmzZtG/f/+Q/RlbOgl0ISLIW6vW2yp/ntdZ08ycrqrB3ozNN30Nk/Q3IifYej1rH9ghlTfeeINf//rXhnP33HMP69ev54EHHmDQoEFN/JMIXyTQhYgwX8vxehum6D5T1P18psXcYK0Tf/yt9e1r6V7nD4DGLB9ss9n4wx/+wIIFCygvL6dbt24MHjzYdT45OZlXX3016LpF48iwRSFigHNmaTBdMU69C98LOtQXjO7t832acm9P5eXlLFy4kPnz52Oz2VzH8/Pz+fjjj73uDiSaRoYtChHjgplZ6qnglsuDGmWS6/YA0xtfI3KCCfOKigrmz5/PJZdcwrRp0wxh3rNnTyZMmBDwPUToSJeLEDHAXz+2L57925kWM2eqawxrpQS7rVpjd2Wqqqpi0aJFPP3003z33XeGc126dKGwsJDRo0djMgU3rFGEhgS6EDEgUD+2L55BHGiFx1AoLy/niiuu4MiRI4bjHTt2ZObMmYwbN47kZImWaJBPXYgY4Gs4Y2M3LW7u/qfByMjIoH///q5Az83NZdq0adx3332kpKSE9d7CP+lDFyIGNKcfO5zq6uooLm7Yj19QUEC7du1YsGABBw4c4KGHHpIwjwEyykUI0YDWmqKiImbMmMG3335LSUlJgx2B7HY7ZrP3vT9F+MgoFyFEULTWrFmzhn79+nH77bezZ88efvjhB/74xz82uFbCPPZIH7oQAoB169Yxbdo0Nm3aZDienp4u4R0nJNCFaOE2bNjA9OnT+eCDDwzHU1NTeeSRR3jiiSfIycmJTnGiUSTQhWihSktLeeCBB3j33XcNx1NSUhg/fjxTp07loosuilJ1oikk0IVoobKystixY4frtclk4r777mPatGl06NAhipWJppKHokK0ELW1xqUF0tLSmDp1KklJSdxzzz0UFxezaNEiCfM4Ji10IRLcgQMHKCws5MyZM6xatcpwbvz48QwdOpQePXpEqToRShLoQiSow4cPM2fOHF555RVX6/yzzz7j6quvdl3TqlUrCfMEIl0uQiSYsrIyHn30Ubp27crLL79s6Gp5++23o1iZCDdpoQuRII4dO8YzzzzDiy++SFVVleHc4MGDmT17Nvn5+VGqTkSCBLoQce7HH39k/vz5LFy4kDNnzhjODRgwgNmzZzNkyBAf3y0SiQS6EHFu//79PP3004Zjffv2Zc6cOQwbNkx2C2pBpA9diDjXv39/RowYAUCvXr0oKipiy5YtDB8+XMK8hZEWuhBxoqqqipdeeom2bdsybtw4w7nf/e53jBs3jl/84hckJUk7raWSQBcixlVXV/PnP/+ZOXPmUFZWRrt27bjjjjuwWM7tZnTllVdy5ZVXRrFKEQvkR7kQMaqmpobFixfTrVs3/uM//oOysjKgfljiK6+8EuXqRCySQBcixtTW1rJ06VIuu+wy7r//fg4fPuw6d+GFF/L8889z//33R7FCEauky0WIGFFXV8eqVauYOXMm+/btM5zLzs5mypQpPPzww6SlpUWpQhHrJNCFiBEvvvgijz32mOGY1Wpl8uTJPPbYYw22gBPCk3S5CBEjxo0bR1ZWFgAZGRnMmDGDkpISpk6dKmEughJUC10pNRz4I2ACXtZaz/VyzS+BAkADO7XWY0JYpxAJ5cMPP6R9+/ZceumlrmOZmZnMmDGD7777jsmTJ9O2bdsoVijiUcBAV0qZgBeAocBRYItSarXWep/bNV2BJ4F8rfUJpdT54SpYiHi2adMmpk+fzvvvv8+YMWNYunSp4fyECROiVJlIBMF0uVwNHNBaH9JaVwPLgVs9rnkQeEFrfQJAa30stGUKEd8+//xzRowYwYABA3j//fcBWLZsGXv27IlyZSKRBBPoucA3bq+POo656wZ0U0ptVEptdnTRNKCUGq+U2qqU2nr8+PGmVSxEHNmzZw+jRo2ib9++rFmzxnU8KSmJe++9l8zMzChWJxJNMIHubTEI7fE6GegK/BS4C3hZKWVt8E1aL9Ja52mt82QXcZHI9u/fz5gxY7jyyisNuwQppRgzZgxffPEFixcv5uKLL45ilSLRBPNQ9Cjg/q+uPVDm5ZrNWms7UKKUKqY+4LeEpEoh4sgzzzzD1KlTqaurMxwfNWoUhYWFXH755VGqTCS6YFroW4CuSqnOSqkU4E5gtcc1RcBgAKVUNvVdMIdCWagQ8aJv376GML/pppv4/PPPeeONNyTMRVgFbKFrrWuUUo8Ca6kftrhYa71XKTUL2Kq1Xu04d71Sah9QC0zWWv8QzsKFiAX/+te/aNu2LcnJ5/5X+tnPfsagQYNISUlh9uzZXHPNNVGsULQkSmvP7vDIyMvL01u3bo3KvYVoru+//5558+bx/PPP89JLL3HPPfcYzp8+fZrWrVtHqTqRyJRS27TWed7OyUxRIRrBZrMxY8YMOnfuzLPPPktlZSWFhYXY7XbDdRLmIhpkLRchglBeXs7ChQuZP38+NpvNcK5NmzaUlZXRsWPHKFUnRD0JdCH8qKio4MUXX+SZZ57h+++/N5zr2bMns2bN4rbbbpNdgkRMkEAXwoc//elPrrVV3HXp0oXCwkJGjx6NyWSKUnVCNCSBLoQP+/fvN4R5x44dmTlzJuPGjTOMahEiVsjviUIA3kZ7/dd//Rfp6em0a9eOF198kf379/OrX/1KwlzELAl00aLV1dWxfPlyevfubdjqDSAnJ4f33nuPAwcO8PDDD5OSkhKlKoUIjgS6aJG01rz55pv07t2bu+66i127djFnzpwG1w0YMACLxRKFCoVoPAl04VXR9lLy566j85S3yZ+7jqLtpdEuKSS01qxZs4Z+/fpx++23s3v3bte5N998k9OnT0exOiGaRwJdNFC0vZTJb+yk1FaJBkptlUx+Y2fch/q6devIz89nxIgRbNu2zXU8PT2dp556iq+++komBIm4JoEuGih8ay/2WuNDQnutpvCtvVGqqHk2bNjA4MGD+dnPfsamTZtcx1NTU5k0aRIlJSXMmTPHtZ+nEPFKHteLBk5U2Bt1PJZprXnkkUfYtWuX61hKSgrjx49n6tSpXHTRRVGsTojQkha6SGhKKQoLCwEwmUw8+OCDfPXVVzz//PMS5iLhSKCLBqwWs89z04p2+zwXbV988QVPPPFEgzHlt956KzNmzKC4uJhFixbRoUOHKFUoRHhJoIsGCm653Oc/jCWbj8RcqB88eJB77rmHK664gmeffZY333zTcN7ZSr/00kujVKEQkSGBLhp4fesR6vycX7r5SMRq8efIkSM8+OCDdO/enVdffdW1S9CMGTO8zvwUItFJoAuDu/+0iY0Hf/R7jYaoDmEsKyvj0UcfpWvXrrz88svU1ta6zg0fPpxXXnkFpbztbS5EYpNAFwaBwtxp3triMFfS0LFjx5g0aRKXXnopL7zwAtXV1a5zgwcPZsOGDbzzzjv069cv4rUJEQtk2KJwaUyru8xWGcZKGjp9+jQ9evTgxIkThuMDBgxg9uzZDBkyJKL1CBGLpIUuXBozcaidNbLrm7Ru3ZrRo0e7Xvft25c1a9awYcMGCXMhHCTQhUtjJg5NHtY9bHWcOXOGDz74oMHxp556in79+vHmm2+yZcsWbrjhBukrF8KNitZogLy8PL1169ao3Fs0VLS9lAkrdgR9fXqKCbMpiZOVdtpZLUwe1p2RfXKbVUNlZSUvvfQSc+fO5fTp0xw6dIgLLrigWe8pRKJRSm3TWud5OyctdAFAwerGrdNyproWW6XdtXjXk6t2N3nkS3V1NS+++CJdunTh8ccf59ixY1RUVDB37twmvZ8QLZUEugDAVtm8dVoq7bWNHvlSU1PD4sWL6datG4888ghlZWWucxdffDG9e/duVk1CtDQyykWETGmQI19qa2tZvnw5BQUFHDhwwHDuwgsv5KmnnuLBBx+kVatW4ShTiIQlgS5CRlHfF++vL/3rr79mxIgR7Nu3z3A8OzubKVOm8PDDD5OWlhbmSoVITNLlIkI261MTeMJR+/btDROCrFYrv/3tbzl06BCTJk2SMBeiGaSFniCKtpcyb20xZbZKw6gTf8cLVu9tdt+5J/cJR1prTp06RWZmputYcnIyhYWFPPTQQ0ycOJGJEyditVpDWoMQLZUMW4wyX4Hb2Pd4ctVuKu3n1jSxmE2M6pvL37aVGo4r6lvS4ZJrtbBxyhA++ugjpk2bRl1dHR9//LFhvHhtbS02m422bduGsRIhEpO/YYvSQo8izyB2Dv8DGhXq89YWG0Ib6kedLPv0G2o9fmCH+8f3LRedZujQofzzn/90HXvvvfcYNmyY67XJZJIwFyIMJNCjyFcQT1q5k4krdgTdYve1ropnmIdT9b8OYvt4CVMObjEcT05OZs+ePYZAF0KEhwR6FAUK4mBb7JkWc8j7woNVffwwJzcspWL/J4bjSUlJ/Pu//zvTp0+nc+fOUalNiJZGAj2CPPvLrWnmgOunVNprmbBiB0+u2kWq2YStwt7g4eaZ6poI/QnOqTl5jBMf/YWKfR9h7MhRZF81hI0rX6Rbt24Rr0uIlkwCPUK89ZebkxRmk8JeG7hrpNJeR6W9zvW9E1bsoGD1XqpraoP6/lDT9rNUfPEx7mGe1m0AmdeNoVVOJwlzIaIgqHHoSqnhSqlipdQBpdQUP9fdoZTSSimvT2BbMm/95fY6TXpKMrlWCwowNXLlQFulnQq7v83iwsecfTHpPQcBYLm0Hxfd+0dybptKSk6niC+tK4SoF7CFrpQyAS8AQ4GjwBal1Gqt9T6P6zKA3wCfhqPQWNHUYYa+psXbKu3smHk9ANOKdrN085Gwj0RpjNrTJzi5eSWmjGwyrxllOGcdOJaMPiNoldvDdcxiNoV1aV0hhG/BdLlcDRzQWh8CUEotB24F9nlcNxt4FvjPkFYYQ5ozzNCklNdRJ85WedH2Uv62rTRmwry24iSnPltF+bZ/oGvOolql0/qqYSSntnbVmJx5AZbzLiQ9JTmky+gKIZommEDPBb5xe30UuMb9AqVUH+BirfU/lFIJG+i+hhnOW1scMMR8DSF0Hvf23tFQV3WaU1uKOLX17+hqt1mfZ89Qse8DMn5yk2tyUq4EuBAxJZhA99ax60onpVQS8Bxwb8A3Umo8MB6gQ4cOwVUYQ3wNMwxmf81cq8Vrt0uuo7850nt0eqo7W0H5trc49dkq6s6eMZxLueBSMgeOxXJJ/aMRZ5hvnCJbvwkRS4IJ9KPAxW6v2wNlbq8zgCuADxzTuy8EViulbtFaG+b2a60XAYugfup/M+qOinY+QjmYh4CTh3VvMD3fbFKcOVtD51VjOXUAABNJSURBVClvk+SjSybc6uxVlH++hlOfvkFd5SnDOXN2B6zXjcXSrT/1P7fPifYPICFEQ8EE+hagq1KqM1AK3AmMcZ7UWp8Esp2vlVIfAP/pGeaJwFsoOx8CBnpY6vzaeY01zczpqhrXhKBohDnAmT3rsH2w2HAsOasd1uvGkNZjICrJ5PX7ZCSLELEnYKBrrWuUUo8CawETsFhrvVcpNQvYqrVeHe4iY4VnKDuDG2jwsHTiih1sPfwjc0b28vpepyprohbi7lr3+jknN62ktvx7TJkXYB1wJ+lXDPEZ5E4ykkWI2COrLYZA/tx1XrtiFHB3/w6s//I4pbbKsK906I+uq+XMvg8wZ3ek1YVdDOfO7PuQurNnaH3lUJTJHPC9rBaza6ilECKyZLXFMPPVn6zBMK48GmGudR0VX27AtuE1an48Smrnn3DBL2cZrnFOEAqGxWyi4JbLQ12mECIEJNBDwNfDUohii1xrKr/ajG3DUuzHv3Ydryr5nKqje0ltH3wom5SiTmsZZy5EjJNAD4HJw7ozccWOmJgUpLWm6tA2bBuWUv3dV4ZzKiWNNlffRkpO41Y/rNOakrkjQlmmECIMJNBDYGSfXLYe/rHBtP1I95lXHd6F7eNXOVv6heG4MqeSkXcLbfrdhsmS0ej3lREtQsQHCfQQmTOyF3kdzzOMgBncI6fBFnC+KJq3rvmJj17l1KYVxvdMTqF1nxvJvOYOTOnB7dtpTlLY6879GJK1WYSIHxLoITSyT26D/mVnyPvqY3fSQHlV09c1T+t27blAT0omo/cw2vT/JckZwW/1lpVmZubNlzd7j1MhRHRIoMeQYMelVx8rITmrHUnmVq5jrS7sQlqPgSSlWMgccCfJmec3+v4zb77c6w8lIUR8kEAPI8/VGZvL/v032Da+RsWXH5M15AHa9BtpOJ99y+QGU/SDNbZ/BwlyIeKcBHoYOJcBCNTNEiz7iW85ufE1zuz7EHT9hhYnN79O66uGkZRy7oFlU8Mc8DmjVQgRPyTQQyyUrfKaU8c4uXE5p3f/0xXkTq1yL6PubIUh0JtqbP/4W/lSCNGQBHozeC7INbhHDss+/abZa7TUlP/Aqc0rKd+5FmqND0pTO/fFOvBuWl0Umj078y89T1rnQiQICfRGcA/wTIuZM9U1rg2aS22VLNl8pNn3sG1cxqnNr6Nrqg3HW3W4EuvAsaS279nsezgtGN1b+s2FSCAS6EEq2l7K5Nd3usZoN3W8eCAqyWQI81a5l9VvLtHxqpDeJyvNLGEuRIKRQA9Sweq9hgk3oaBr7Khk4+qGGX1v5tTWv5PcJgfrwHGkdv4Jjo1DQmrmzbLAlhCJRgI9SKFskddVV1H++T849dkqLrjrd6TkdHKdS0qxcOG435OceUFYghxkiKIQiUoCPYJ0TTXl29/h5ObXqauwAXByw2vk3DbVcJ3ZemHYashKM8tDUCESlAS6H+4PQZWCpg5e0bV2Tu/6X05+soLa0z8YzlUfK6kfftgqLQQV+2cxm6SrRYgEJoHuQ4Px5E0Ic11Xy5k972PbuJzaU8cM50wZOWQOGE3rXj9HmcL/1+Bcp0W6WoRIXBLobtxb5ElKNXk8udaaii8+rN8l6ESZ4ZwpPYs21/6SjKuGN3ggGi5j+3eQbhYhWgAJdAfPFnlzJweV73jXEOZJljZk9r+D1n1uJMmc2qz3DpZzT1MJcyFaBgl06sN80sqdzQ5xJ6UU1oFj+ddrU0hqlU6bq28no+/NEeknd5IuFiFanhYf6M6WeVPCXGtN1dfbOb3zPbJv/k9DX3jqxVfQ9ob/Q1q3a0lKbR3Kkv1SwHMyA1SIFqnFB/q8tcVNWkir6ps92D56lbNH9wJwulNvMnoPN1zT+sqhIakxWBaziadv7yVhLkQL1aIDvWh7aaOXuD1b+iW2j5dQdXiH4fipz1bR+qrrm7WEbVOYHA9vc2V3ISFavBYZ6EXbSylYvbdRsz+r/3UQ28dLqDy4xXgiKZnWV11PZv9fRjzMAQ4+fWPE7ymEiE0tLtAbu1559fHDnNywlIr9nxhPqCTSrxhC5oA7wzqz05+stMgMexRCxIcWF+iN6TOvq67iuyWT0dUVbkcVaT3/DWv+GMznRa97w2xSMutTCGHQ4gK9rBF95kkpqfWrH25aAUBatwFkXnc3KTkdw1VeA87hh4BhMw3pLxdCeGpxgd7OavH6ILTm1PdUfbOb1pcPNhxvc/Vt1JwoI7P/HaRccGmkyvQ6YkUCXAjhT4sL9MnDuhtnhJ45wclNr1O+4x2oq6VVux6Ysy5yXW9KbU3OrU+Eva6sNDNaw8lKu7TAhRBN0uIC3RmSv1v1GcX/u5Tyz/+Btp91nT/5yTKyRzwe0ZpMSrF9xvURvacQIvG0uEC32Wx8/uZ/s+e556g8c9pwLuWibqT3HOzjO8Pnrmsujvg9hRCJp8UEenl5OQsXLmT+/PnYbDbDOfP5l2AdOBbLpf3CtkuQ615JUKuhTte3zO+65mJZPEsIERItItAPHTpEn779OGX70XDc3LYDmQPvJq3btWGfFCQzOYUQ4dYiAn2nLYWzqecB9YGenHURmfljSL/s31BJprDfP9dqYeOUIWG/jxCiZQuqWaqUGq6UKlZKHVBKTfFy/nGl1D6l1C6l1PtKqcgN1PZgt9spKSkxHJv/3n4yrxuLqc35tL3hN7R74CVaXz44ImGeRP3IGiGECLeAga6UMgEvADcAPYG7lFI9PS7bDuRpra8E3gCeDXWhgdTU1PCXv/yFHj16cPPNN1Nbe242aJmtktRL+pI7/r9pfeX1EQlyp8w0s3SzCCEiIpgul6uBA1rrQwBKqeXArcA+5wVa6/Vu128GxoaySH/q6upYuXIlBQUFFBcXu45fdvdM7J2upZ3VQqbFXL8Qlynya5+cqAhuATD37e9kHLoQoimCCfRc4Bu310eBa/xcfz/wjrcTSqnxwHiADh06BFmid1pr/v73vzNjxgx2795tOJdkyeDHU6dpDZTaKjElhXfkij+mIEbNeC4YVmqr5MlV9X8mCXUhRLCCCXRvieR1ex+l1FggDxjk7bzWehGwCCAvL69J+71prXn33XeZPn0627ZtM5xr06YNmVffBleMMGz3VlsXmq3lmiKYnZC8LRhWaa9l3tpiCXQhRNCCeSh6FHCf+dIeKPO8SCn1c+Ap4Bat9VnP86Gwd+9errvuOm688UZDmKenpzN16lRKSkow9f1FRPfuDMRqMZM/dx2dp7xN/tx1FG0vbXCNrwXDGrOQmBBCBBPoW4CuSqnOSqkU4E5gtfsFSqk+wH9TH+bHQl9mvczMTEOQp6amMmnSJEpKSvjtb3/LeeedRzurJVy3bzRzkuJMdQ2ltko09V0pE1fsoJNHuPuqOZb+LEKI2Bcw0LXWNcCjwFrgC2Cl1nqvUmqWUuoWx2XzgNbA60qpHUqp1T7erlHq6uoMr9u3b89DDz2E2Wzm0Ucf5eDBg8yfP5+cnBzXNYN75Hi+TcSkmZPISjOjqB973jo1GXutscvF+crZT160vZTJw7pjMRtH3ljMJhnuKIRoFKWbsNt9KOTl5emtW7d6Pbdr1y5mzJjBJZdcwh/+8AfDue+//54zZ87QsWPDoe7TinazZPORsNQbSFaaucECW52nvO39YYMb56QjGeUihAiGUmqb1jrP27mYmin6xRdfUFBQwMqVKwFo1aoVkyZNIjf3XLBlZ2eTnZ3d4HujGeYWs8nr7kG+1l535+wnH9knVwJcCNEskd/V2IuDBw9yzz33cMUVV7jCHODs2bO8++67Qb3Ha5+GJsydQ3pyrRYWjO7NgtG9sVoajl93v85zIwonb10pnqSfXAgRKlFtoR85coTZs2fzyiuvGGZ2Atx6660UFhZy1VVX+X0PZ1dFqEYmahquvTJvbXH9xCQ/1/nrMpm3tphSWyUK43hP6ScXQoRS1AL9yJEjdO3alerqasPx4cOHM2vWLPr16xfwPaYV7Wbp5iMB+6kby3O4YKBhhYEmBjmDXfrJhRDhFLVAP378uOH1T3/6U+bMmUN+fn5Q31+0vTQsYQ4Nu0F89YU7rwt2YpD0kwshwilqfeipqakAXHvttbz//vusX78+6DCH+hANR5h76wYJNKxQJgYJIWJB1FroHTp0YMGCBQwfPrxJuwQ1NiytFjOXt8vgk4M/NvhB4Ozb9tyEwr2LJNNiJtWchK2i4SbOgVrwQggRCVEL9IyMDG644YYmf38wQwLdFdxyOSP75Abdj+3ZL26rtGMxm3hudO8G1w/ukdOg+0ceeAohIi2mxqE3xuRh3Q2BG4i3h5T+BNsvXrS9lL9tK23Q6v9Jh0zpLxdCRFRMjENvipF9cnn69l7kWi2uqfZj+3fwOe7bGcbB8tWlU2qrNCyw5S34AT45+KPhuqLtpQEX6RJCiOaI2xY6eG9t53U8jwkrdni9vjFdNP66dNxb+76CX4OrNS/rnQshIiFuW+i+jOyT63VmJ9Q//Ay2Zexvlqd7a9/fg09n2PvrvhFCiFBJuECfVrS7waxOJ2erORjOLh1fnGE9eVh3rzuAwLmwl2GNQohISKhAd0428qcxITqyTy65AdYqH9knl7v7d2gQ6u6jXGS9cyFEJCRUoAcz2aixIRrMWuVzRvbiudG9DQ9o3RfskvXOhRCRENcPRT0Fan2bTYozZ2voPOXtoNdScV9gy9/YdX/DIYN9DyGEaI6Y3OCiqfLnrvM5MiXNnIS9VmN3W5bRYjb5XPo2kmTRLiFEsPxtcJFQXS7eujYUMLZ/B7LSWxnCHGJjpIlzSKP7vqPOremEEKIxEirQvU02em50b+aM7BWzI01kSKMQIlQSqg8dfPdlx+oCWrH6g0YIEX8SqoXuT6yONJEhjUKIUGkxge6tOyYWHojG6g8aIUT8SbguF39icccgGdIohAiVFhXosSoWf9AIIeJPi+lyEUKIRCeBLoQQCUICXQghEoQEuhBCJAgJdCGESBAS6EIIkSAk0IUQIkFIoAshRIKI2nroSqnjwOEwvHU28H0Y3jfU4qVOiJ9a46VOiJ9a46VOiJ9am1tnR611jrcTUQv0cFFKbfW1+HssiZc6IX5qjZc6IX5qjZc6IX5qDWed0uUihBAJQgJdCCESRCIG+qJoFxCkeKkT4qfWeKkT4qfWeKkT4qfWsNWZcH3oQgjRUiViC10IIVokCXQhhEgQcRnoSqnhSqlipdQBpdQUL+cfV0rtU0rtUkq9r5TqGI06HbUEqvUhpdRupdQOpdQGpVTPaNTpqMVvrW7X3aGU0kqpqAwRC+IzvVcpddzxme5QSj0QjTodtQT8TJVSv3T8e92rlHot0jU6agj0mT7n9nnuV0rZolGno5ZAtXZQSq1XSm13ZMCNMVpnR0c+7VJKfaCUat/sm2qt4+o/wAQcBC4BUoCdQE+PawYDaY6vHwZWxHCtbdy+vgV4N1ZrdVyXAXwEbAbyYrFO4F7g/0bjc2xCrV2B7UCW4/X5sVinx/WPAYtj+DNdBDzs+Lon8HWM1vk68O+Or4cArzb3vvHYQr8aOKC1PqS1rgaWA7e6X6C1Xq+1rnC83Aw0/ydf0wRT6ym3l+lAtJ5SB6zVYTbwLFAVyeLcBFtnLAim1geBF7TWJwC01sciXCM0/jO9C1gWkcoaCqZWDbRxfJ0JlEWwPqdg6uwJvO/4er2X840Wj4GeC3zj9vqo45gv9wPvhLUi34KqVSn1iFLqIPVB+ZsI1eYpYK1KqT7AxVrrf0SyMA/B/v2Pcvwq+4ZS6uLIlNZAMLV2A7oppTYqpTYrpYZHrLpzgv5/ytF92RlYF4G6vAmm1gJgrFLqKLCG+t8oIi2YOncCoxxf3wZkKKXaNuem8Rjoyssxr61apdRYIA+YF9aKfAuqVq31C1rrS4EngGlhr8o7v7UqpZKA54BJEavIu2A+07eATlrrK4F/An8Je1XeBVNrMvXdLj+lvuX7slLKGua6PAX9/xRwJ/CG1ro2jPX4E0ytdwH/o7VuD9wIvOr49xtJwdT5n8AgpdR2YBBQCtQ056bxGOhHAfcWV3u8/EqllPo58BRwi9b6bIRq8xRUrW6WAyPDWpFvgWrNAK4APlBKfQ30B1ZH4cFowM9Ua/2D29/5n4C+EarNUzB//0eBv2ut7VrrEqCY+oCPpMb8O72T6HW3QHC13g+sBNBabwJSqV8QK5KC+XdaprW+XWvdh/qsQmt9sll3jcaDjWY+bEgGDlH/a5/zYcPlHtf0of6BRNc4qLWr29c3A1tjtVaP6z8gOg9Fg/lML3L7+jZgc6x+psBw4C+Or7Op/zW9bazV6biuO/A1jgmJMfyZvgPc6/j6MuqDNKI1B1lnNpDk+Pq3wKxm3zdafzHN/LBuBPY7Qvspx7FZ1LfGof7X7H8BOxz/rY7hWv8I7HXUud5fiEa7Vo9roxLoQX6mTzs+052Oz7RHrH6m1P9q/gdgH7AbuDMW63S8LgDmRuuzbMRn2hPY6Pj73wFcH6N13gF85bjmZaBVc+8pU/+FECJBxGMfuhBCCC8k0IUQIkFIoAshRIKQQBdCiAQhgS6EEAlCAl0IIRKEBLoQQiSI/w/KbWLYc9xzFQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(preds_df3[(preds_df3['actual']==1) & (preds_df3['t1']!=preds_df3['t2'])]['t2_attn'], preds_df3[(preds_df3['actual']==1) & (preds_df3['t1']!=preds_df3['t2'])]['t2_cont'])\n",
    "xpoints = ypoints = plt.xlim()\n",
    "plt.plot(xpoints, ypoints, linestyle='--', color='k', lw=3, scalex=False, scaley=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 354,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1a8a276750>]"
      ]
     },
     "execution_count": 354,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD5CAYAAAA3Os7hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO2de3xT9f3/X++kpyXllhYYQqDARGEyhM4KKO6r4IYoQ6soiKDzMvl6mf4E160qPwF/Kv3aL+J1U2F+NwUREdeBsOEFFOVrFVhbGEqdggKBKZeWSxvaNPn8/khOepKcy+ckJ5cmn+fjgTYnJ+e8c5K8zufz/rwvxBiDQCAQCDo+tlQbIBAIBAJrEIIuEAgEGYIQdIFAIMgQhKALBAJBhiAEXSAQCDIEIegCgUCQIeQY7UBELwP4BYDvGWM/VnmeADwN4AoAzQBuZoz9w+i4PXv2ZAMHDjRtsEAgEGQz27dvP8IY66X2nKGgA/gTgOcAvKLx/OUAzgr+Gw3gD8H/6zJw4EBs27aN4/QCgUAgkCGib7WeM3S5MMY2Azims8tVAF5hAaoBOImoj3kzBQKBQBAPVvjQXQD2Kx4fCG6LgohmEdE2Itp2+PBhC04tEAgEAhkrBJ1UtqnWE2CMvcQYK2GMlfTqpeoCEggEAkGMWCHoBwD0VzzuB+CgBccVCAQCgQmsEPQ1AG6iAGMAHGeMHbLguAKBQCAwAU/Y4goAlwDoSUQHAMwDIAEAY+wFAOsRCFn8CoGwxVsSZaxAIBAItDEUdMbYdIPnGYC7LbNIIBAIBDEhMkUFAoEgQxCCLhAIBBmCEHSBQCDIEHhS/wUCgSDrqKpxo3JDPQ42etDX6UDZZUNQWqyaM5k2CEEXCATcdESRi4WqGjceeGsnPF4fAMDd6MEDb+0EgLR+v8LlIhAIuJBFzt3oAUO7yFXVuFNtmuVUbqgPibmMx+tD5Yb6FFnEhxihCwQCLvRELp1HrbHMKg42ekxtTxeEoAsEAi7cGmKmtT0diNV10tfpUH1ffZ0O1XOkixtKuFwEgjSmqsaNsRUbMah8HcZWbEype8NOanX4tLcDqbf/ob/sjMl1UnbZEDgke9g2h2RH2WVDwralmxtKjNAFgjQlltGl1mjRilGkj6kWUdXcnuqFxblVO9HU6lN9zt3oQVWNW9MOebvRNTNyQyV79C4EXSBIU8z6rLUEdNu3x7B6uztuYXVpuCFcKm6IWOzXwkgUtZ5f8el+naPC8BqUFrvi8rWn4oYmBF0gSCJmRmxmF+a0BHTFp/ujRtGxCGvZZUPCBApQd0PIWOFzVxPF+1bWYsHaXZg3eRgAaIqm1sxBxuP14f436jB7ZS2c+RIYA457vKZG0t0dEho9XtXtqVhEFj50gSBJmPW3qi3A6W3XEnotYZPdDnr2Kv3fALDwmuFwOR0gBEbmC68ZHiVO8uv04PWnq4kiADQ0e/HAWzsxf80uVdGc/Uat4bGBwLVhweM1erym/eBaywdE1kbKtLS0cO0nRugCQRyYGXGbHbGZHRFrRWboIY9mZfvk9zFuaK8oN819K2vhdEiYf+UwXR9+pM1q8Lof9MTP4/VpnsdgcG4I70i6sTl6dC5vNxMpo8fmzZtx4403YunSpfj5z3+uu68QdIEgRsz6SHlHbMqbRHeHhE6SLSQQejcMtRuAER6vDwvW7sJprz/sfSyv3qfaR7LR41V9j7LNZm4oWqKpfP82IkPXSaLQ+1zkm56WffJnZeaGHInX68UjjzyCxx57DIwx3HTTTdixY4fua4SgCwQmMBIbpV9W/lEDgdGvliwpR2yRN4lGjxcOyY7F00Yajhbl5+9byedukGlQGWXqSWikEPOOytVwN3owtmJjWDSO8lhGYl6QL4XdjKyku0MK/a12815WvU/1dbJo80bKqLFnzx7MmDED1dXVoW2tra348ssvdV9HLEV3v5KSErZt27aUnFsgiIW5VTs1R66x4pDsYX7osRUbVUe5diIsmjoiTES1hELrGFYzc0wRNu0+bMm5JBuhS6cc1ZuLFoTAjadAsaDJO6K32wg+v/HNoubhCQD4r2nk5xQLy5Ytw1133YWTJ0+Gto0fPx6vvPIKXC4XiGg7Y6xE7bViUVQg4KCqxm25mDuD7pTZK2tDi4R6C5vyQp3R4qpaUowe2mlB+iyr3mfZjcPrZ6bEHGifRTQ0e3Hc48WMMUVYNHUEJLvxO8rj2EfpH+ddyPQzFrOYnzhxAjNnzsSNN94YEvOcnBz813/9F9599124XMbHFS4XgUCHWHzDvLS0+aP87858SVPYZFdHc2ub7uKqLCj3v1GnOlq1EaAcnKrdpAI3BAaP1x/Te0s2DMDy6n0oGVCIymtHYMHaXbo3iGaO96V0hWmFJ+q9xgzV1dW44YYbsHfv3tC2wYMH47XXXsP555/PfRwxQhcINFCOhK3GTqQqyoxBd3TtbvRoCpVyFFla7IJfw/Vg4GlAvmRDXo6tw4i5DANCN7Wahyfgm4pJmDmmKKZjRS5e6lQ30HwNLydOnMDll18eJua33HILampqTIk5IEbogixDz/cc+ZzaSNgKHJJd87jHPV4snjZSc3Rt1/ERMwDFj7xj2p8cicfr5xrBpiORKf2Plg437SpzqSxeaoUnytiJVGPyeejWrRsWL16MW265Bd27d8cLL7yA66+/3vRxACHogixCK+tw1bZ9uK6kKOq5RFCQL2He5GGabpy+TkdIFNRC3oxuMMrRe6zhfqkJk7COslV1ANqjfnjj8yUbofI69QVNo2PEuxD6y1/+EgcOHMDMmTMxcODAmI/D5XIhoolEVE9EXxFRucrzA4jofSLaQUQfEFG/mC0SCDgxW8lPK+twy9fHMHtlbUJG41p2uBs9UYuRBGBgDwfGVmzE7JW16CTZ4HRIYVmZWnVTMhHJFttyrdfPwqop8i4SR75OSdllQzQXjwvyJW4xb2pqwp133om6urqw7USEuXPnxiXmAMcInYjsAJ4H8HMABwBsJaI1jLHPFbv9N4BXGGN/JqLxABYCuDEuywQCHXiSeqpq3IaLYzLJGpU2NHtD9kSekyFwc1Hu65DsmBEMD5Rrjkg2gtfIER6BHOLXoaCAWJqNfgHCZ1jKeHCjkbpWNEtpsQvbvj0W5b5xSPZQTRkj/vGPf+CGG25AfX09PvzwQ2zbtg35+flcr+WFZ4Q+CsBXjLE9jLFWAK8DuCpin3MAvB/8e5PK8wKBpRi1CKuqcaPszbqYxCCd8Hh9ofBAueaIjzE4FUkvPHQ4MQfg9TEwBq4wxEgICJuxlRa7sKV8vOEMRy9K5dHS4Vg8baRhLZtI/H4/Fi1ahDFjxqC+PvD9/OKLL7BixQru98MLjw/dBUBZh/IAgNER+9QBmALgaQBXA+hKRD0YY0eVOxHRLACzAKCoKLbVZ4EAME6jr9xQD6+vI8qYMX4GNLV07BsVL40eL8aeWRg2c+GBIRC2KSMvduvNcHiiVHhK6io5dOgQfvnLX+Ldd98NbevSpQuef/553Hij9U4MHkFXuz1GXo3fAHiOiG4GsBmAG0Bb1IsYewnAS0AgU9SUpYKsRCsqRWuRykaEQeXr0mJESgCc+VKoDktTSxtXLDMPyQxCsROQynujWTGX8TGGsjfrAIaQgDc0eyHZCc5gXLkcNaQW2RIvb7/9Nm655RYcOXIktK2kpAQrVqzA4MGDLTuPEh5BPwCgv+JxPwAHlTswxg4CuAYAiKgLgCmMseNWGSnITvT85FqFqFJVyEkNBuC01x+qw1JV48aclbXoSAGBkp1Qee0I0/Vh0gW1WZrXx9A5Lwe18yYk5Jwejwe//e1v8dxzz4W2ERF+97vfYcGCBcjNzU3IeQE+H/pWAGcR0SAiygVwPYA1yh2IqCcRycd6AMDL1popyEaMys0qa3Pr9bXUI9a0d17kYl2yP7cjibnL6UDltYFwvEyLrjGqBR8rXq8XF154YZiY9+3bF++99x4WLlyYUDEHOASdMdYG4NcANgD4AsAbjLFdRPQIEV0Z3O0SAPVE9CWA3gAeS5C9giyhqsatGZEg+8nlha69FZM0syKNkIs7JRK5DsuCtbsSeh4r6Zxrx5by8SEXRCwZkOlOIpo5S5KEa665JvS4tLQUO3bswPjx4y09jxZciUWMsfUA1kdse1jx95sA3rTWNEG2IrtatFCLRDCqtaEXtpeMSBi9Zgx62JCaUf1jVw8Pe1xa7MIDb+3ocOUAJDvB52Oq11CePQHW9vh88MEHsWXLFlx99dWYNWsWKMbZYyyITFFBytBa8NRKAALUIxGqatxoao1agw+DIbooVUcgXcytqnGjpS2xYu4KNo1Y+dl+rjh7nnj8XLsNrcwHv4bp8uwJiE3U33//fZx99tno3799mdFut+Nvf/tbUoVcRgi6IKkoqxcqR83KBU+9UqVy3G8sXW1SLY6xJPekyub5a3aFCVzlhnrdm6FksybyZnn1vrCGzbrvn0Mvm1qNZ0WxNG5ubW3F3LlzUVlZiYsvvhjvv/8+7Pb2bNRUiDkgqi0Kkkhk9cLIH6v8w9JK7nAF65xE1gPnjWzR280hJf6nkOobihkaPd5QaYWB5esMMyy7dJJMJztFokyeamnzh5J41LATWZpnYKZx85dffokLL7wQlZWVAIAPP/wQixcvtsyWeBCCLkgaeq4UmYONHtXaG0pXC89xzNLJREOIbMFM6eCGZi+G9e1q2bk9Xh/uW1mLppa2qExRgvXhqTx1zBljePnll1FcXIzt27eHtk+cODEhSUKxIARdkDR4RkFyTesp57k0U6zNjKZ4sNuow5cIsBoCTN80Y00A0qPR44XXx0L1yBNRk4YnQ7ShoQFTp07FbbfdhubmZgBAbm4uFi9ejHXr1qF3794WWxUbwocuSBq8ZUzdjR6s3u5WrZNRVePW9JnbieBnLNSRnbffpc/PdOuMZyPJuhJ2IuTYyXDBlTFrxVw+Fk+G6ObNmzFz5kzs399eAWXo0KFYsWIFRo4caZFF1iAEXZA01LI7tX6kagtVsu9cTXiVzZaVC6a8Qi3EPPkQAnXEZ3NmoVrxCRGAvRWT+M7HGB5++GE8/vjj8CvCZP7zP/8TTz75pOWVEq1AuFwESSMyu9PldGDxtJGawQqRrhUt37myW0ysC6axZpoKYmfGmCKUFruSulhspucnEeG7774LiXlhYSHeeustvPDCC2kp5oAYoQssRK+9m0xktTo9F0rkj0/Ld67stK4n+n7G4MyXcOp0W1j8Mk8nIIG1SDZCyYBCAPpt9awklp6fixcvxubNm9GnTx+8+uqr6NcvvXv3CEHPEHjENNHnN2o4EWlnd4eEptY2TRdK5I9PywevFH490Zen2pHXatzQXljx6X7TPTwFseP1M8xeWYsFa3dxX994fOg8vvKTJ0+CMYZu3bqFtnXu3BkbN25E7969w+LM0xViKfqylpSUsG3btqXk3JlGpJgC4T7lZDC2YqOq2DodEjrn5YQJuFH8sJ1ItUej3LRC+Xq5GqC8r5YdBfkS8nNzom54atdOkH64nA4M7OEwHUnD+zv47LPPMH36dIwePRrLly9PWWIQD0S0nTFWovac8KFnAEbde5KBVjRJo8cb8mfLIWhGKF0oUUS83OdjWLB2FwaWr8OZD6xX7dUp2QmnTreF7JBnD/JIXU/MhW89NoisrWRZdtkQfHOUL1xVbkXK01HI5/Ph8ccfx9ixY7Fnzx6sWLECy5Yts8LklCAEPQMw6t6TDKwUPq2Fq8oN9VG1O/xoL64lT92Ve9iJVGt+yDc8o7BG4W4xj0OyY/HUkZgxxrquZLKLjAc/a3fZ6Yn5/v37cemll+Khhx5CW1ugFlC3bt3QqVMnS2xOBcKHngHw+JYTjVXC55DsGDe0F8ZWbIxyj8Ryg/Ixhmavum2R9WQE8aP0VctiqrU+YQa3iRBUwLg+y1tvvYVf/epXaGhoCG278MILsXz5cgwcODAuW1OJGKFnAEap8lYi1/cYVL4OYys2hupJx9oAQbIRCvKlUBjjlPNcWL3dreoesfoGRSTE3EoK8qWwtYmxFRuxvHofzujeCU9NG4mZY4ricsOYvSmoDQCampowa9YsTJkyJSTmNpsN8+bNw4cfftihxRwQi6IZQzKiXPQWXwFwLy7aKJD5p2an1qKmPPIrW1XHVVqVxwYrSun27pqLY01eS2zKBCQ7Ydr5/bF6uzvsuyDZKay3ZzJwOR3YUt7eWKKmpgbTp09HfX372lJRURGWL1+Oiy66KGl2xYveoqhwuWQIZruRx4Le4qv8w7n/jTrDkRRj2tl6husBFrnqrZKV7062wm4TC6cyXh/Da5/ui7pZWlkZkQe1GeqCBQvCxHzq1Kl48cUX4XQ6k2pbIhGCnubwjLyTFYNuJLbyOY1G6k6dlm966wGVG+p1hUEedfP4xa2cmPrE6DyMeC6HjYC8HJupzkh2Ioz5YQG+OerR/Q28+OKLqK6uxqlTp/Dss8/i5ptvTuvwxFgQgp7G8CTr8Cb0WAHP4qsyY1MrguTU6TZU1bhV7VOr9yKPtoxqfvhZIO79FyP6YNPuw6EbjZDbjoOfwXSbOx9j+Me+41EhioyxMMHu3bs3Vq1ahd69e+Pss8+2zOZ0QiyKpjE88eXJjEHnXXyVmzd/UzFJtemB18807VPWewECoy/5/XTnaKDQ6PFi9XY3yi4bgr0Vk3RnA0YU5EuhujPxNm8QJBaP1xdqwn369Gnce++9KC8vj9rvpz/9acaKOSAEPa3hiS9PZgy6WnEto8SN4xqNm/XsKy12hW4esj/e3ehBU2sbJA5/tfKG1hhHnfNJ5/bBlvLx2FsxCfOvHCZ85XESb5SLEQ3NXjzz5kaMGjUKzz77LJ544gm89957CTxj+iFcLmkMj4sj2THoZhdfY7VPbebh9bFQCr9RQpC70RMKdeTtuhPJpt2Hw+xJlq88E2PjXU4HHi0NREPJcel2IthtQKsFC6aMMZyqWY/Zi/4If1traPuqVavws5/9LO7jdxS4RuhENJGI6onoKyKKmscQURERbSKiGiLaQURXWG9q9qHl4pATbwaVr1Nt0ZWoGPRYiDVGXmsE39jsxZby8Xhq2sio40bywFs7MW5oL8P9eGxIZtZtOom57HaKBwIwbmgvVNW4sXq7OzTr8jFmiZj7mo/j8FuP4ti7fwiJeadOnfD73/8eL7zwQtzH70gYCjoR2QE8D+ByAOcAmE5E50TsNhfAG4yxYgDXA/i91YZmI2oujsjEm0aPF2AIS85JZlEuI2Jx0wDaI3h5u3zcAh0fucfrw6bdh8POb6ZEQeRMKNtwSHbMmzwsdAONHDjwwgCs3u7GgrW7LC+C5vmmFof+5x54vvo0tG348OHYtm0b7rzzzoyLYjGCx+UyCsBXjLE9AEBErwO4CsDnin0YALnmZHcAB600MpuJdHGMrdgY7YrwM+Tn5qDm4QnJNo+LWGLk9aJdIo9bVePGfRoRMO5GT9j5B5Wv4zq/ZKOwc5VdNkTzHJkIETDlvPbrK4eMxlpO2OP1WSrmzOdF4+ZXceKzt8K2T5p+K958+fkOXY8lHnhcLi4A+xWPDwS3KZkPYCYRHQCwHsA9agciollEtI2Ith0+fFhtF4EB6VCIKxmYGdmXFrs03QIEhMoTAOCKlCEAlddFl+/lIVMGhIwBy6r3YcaST0IdoICAmyQd3uLRDc+HibktvzseevrPePu1P2atmAN8I3S1zy/yFj0dwJ8YY4uI6AIArxLRjxljYQGljLGXALwEBFL/YzE420mHQlzJgmdkL48etRY+GRAq0lRV40ZTaxv3uXmOr0SyESqvG8G9f0dArf54PD9cyU4xZ406HRKIAtEs3Udfi+bdH4F5W9D5hz/Bsy8sxS0/L47DssyAR9APAOiveNwP0S6V2wBMBADG2CdE1AlATwDfW2GkoB0eV0S2UFXj5qrtIs9e5q/ZxSUmfZ0OVNUEfL4NJsIeu3TKCd0IIhtxCAJ0zs0JrPuYxCHZMf/KYQoXkAMtP78TXakFixY8gGvO6298kCyAR9C3AjiLiAYBcCOw6HlDxD77AFwK4E9E9CMAnQAIn0oCUGZipqrdXLowf80urmJPskDzCIkcRRRLFyM55l3+LMzeELKB4x4vXCZDSbs2/Avj+xFKiycCUMzcFIW3BAEMBZ0x1kZEvwawAYAdwMuMsV1E9AiAbYyxNQDuB7CEiGYjMCO7maWqjGMWkIxCXB0BHoGWQ+Z4M2ennOfCpt2HY1rAc0jtS1LKBdt0bHHndEhoafPHbFdu0HVi9kfuDJbY5bkmzO/D8S2vY3/1SvxLkvCr0vEYPnx4TPZmC6J8bgeAt/hWpJvA6ZBC09REnTOZRNrEO8pzSHZu4XI5HTgYDAk1CxGwd2F4FUmtcsCx0jnXjqZWX1zJR8qSx/PX7IrJBUIAZowpCksSGvPDAvxj33Hda20j4MmpIwG0zzK7OyScOO0NK+rlbfw3jq79b7Qc3B3a9tOf/hSbN282bWumIcrndiDUOtIra0trFd9Sa6Dc6PGibFVd1L48NiSr4Fc8NvHi8fq4w+3M3iyUqB3equgjGwHdOkkhl8W4ob2wafdh03YqOwpV1bjR0mauEJYMA7C8el/opiIXyJJnOFqF0fws4IqqeXhC2AxGKeZNn3+IoxueB2ttDm37cckFeO2112KyNZsQtVzSCPnLrezWs7x6H1fxLa3SsnqFsLRIh6bTkRg1czbCxxhXxqgzX0JTC18kTCRqSUtWRR/ZidDo8Ya+F6u3uzFuaC9Tx5DFvHJDPQaVr8P9b9TFdU0jv20erw8rPt0fuilq3T6V6wrKz9Xf0owj657EkbWV7WJONjj/4yZ0vXoB+vXrF7Ot2YIQ9DRCTbS0fhSRIz+9kaDZUWI6xrrrnVuOVdfLGi3Il8KqOKphtxFOnW6LckF0zrUHQuaA0P/VmD46OtJCrfSBGQgB37xak+vl1ftMHUde7JUHDIlogO1jLHTT0UNuUSfv13KwHof+dC+a/rkxtE+O8wycMbMS3S+YikMnWrUOJVAgXC5phBnBjBz56bkJzI4SkxHrbtZHr2WTss3Y2IqNmlEljIUvJgdmQztCtbdtBOTZCc0qtbid+blh51DzOedLtlDxKSVaUUlAdCMQh2QPc1nI+2rVgTcjxzPGFMW82JsIZq+sDdl/svZvOPbuC4C/3bbOPx6Pwp/dAVtePoDMzLNIBGKEnkZofWkjR4SEwAhI2aS57LIhqrU2IlPYeUh002k119IDb+3E3Kqdqg2otWySR50yejfERo837HjtRwjgZ1AV88jjap1DrymDXB9+b8UkbCkfj9Jil6Zba9Puwyi7bAj6BhdnKzfUx1XTHQjcrEoGFKZVNrHyZiT1LAotQFBuPnpO/g16TpoTEnMCsjLPIhaEoKcRWkI6Y0xRyFWgjG6QhVDu/lN57Ygwt4PTIcWUwh5rQS1etMRsefW+KJGXRbi02IUp57nCbm5y0Sd5H6NRnPJ4ZnzyPEW6zI4gtcRVft/K63DqdHRFTTP4WeD9xjrKTXQZ+E79hqH7hdOQ23cI+tzyDDqfc0nY8zPGFKU8wqqjIMIWk4AZ94LevlohcJHdzdOdQeXruN0FkS4VvffPkzkq78trgxzip9XyT20fNSI/1+bWNlX3kFY0jtMhxRReKEMAFk8bqenmUUZSJRJ/62l4jx1A3hmDw7azoLuFbO0DGjk0Us2Vlc2IsMUUYjYEMDJpSF480ouNTvRU2uqYdDNhgWa6M8k26cVWy/tq2UDUHn6oFsdvlKmrdq0ARH0HJBtF1TXRi5ePJcNSSV+nI6rfq9zeb9PuwyHffSJr0LR+tweH1zwBf/Nx9Ln1WeR07Rl6TinkMounjRQjc5MIl0uCiScEMNLXrEUiF4y0/N3R/mh+tPzhaph1d5QWu1A7b4JmNIu8r1b0iXJwrBWjreYTB7Sv1fw10XXAvX6Gzrk5UW4tPbuNImYK8iXVFn2SPbCOoiw0RkBYez+5D2siYMyPE1urcOjVOWg7dgD+0ydx5O0nEVG7LwyX4gYk4EcIeoKJJwSQx8+b6MJciYhJV/PRzxhTZLgQa2ax1mjfSBvUYsjNvk+ta6U1Wzju8UbdGPTs1mvqQQj0QK28bkRYQ+uCfAmV144AgLAyuGox5JUb6k01AOHBd6oB36+aj4aNSwFfIL6fpE7oMmwctG7j8g1IYB7hckkwZkIAzaS2E2CJ+8OIRMWkq9WjKRlQqOvaMVOYjGdfnsYXZt6n2Wui9h0wslu2eW7VzrBMTXmBuGRAIWrnRTc6UWuMEknAFQRYVSTS8/VWHFn/NPzNjaFtuWcMRs/JZZAKtb+zldfGVoteIAQ94fCWu1XztWvV60jmImgy66/zFB0zU5jMzL5a79NGFIoiivUYaujNrHjs3rT7sOYoW+21vDcbnehLblhbKxo++B+c3L42bHu30VPg/OlMkF07DFO4WuJDuFwSDG8IoFaWaOSkVLITmlraVGO1E0GiY9LTBS3/tI8x7jUDnqxQq8JAzc6ckpWY03r4Wxx6ZU6YmNu7FOIH0x5FwSW36Iq5jUS8ebyIEXoS4Blxaf0QGdorADrzpbDU9GQUzVKr7Z2Xk3njAPl93v9GXVTYoN7Il/cYgLUzK7MzJ96StfHiPbof3sPfhB47Bo9Gj8vvhT2/u+Frn5wqolriJfN+mR0UrR+iLAJ7KyYhPzdHtaZHMopmnVbMxRs93rgjXdKR0mIX/Bp5GWb84107RY+TrJ7VqM0GJBuhuVV99qacKSaSzkMvQufhPwfl5KJwwl3odc1cQzGXbISnRIiiJQhBTwBy7LgZtwiPayMVRbOqatyqVfk8Xh8WrN2VsPOmingyQeV1kMioFrkwmJWCFenKczokINhvUyu8VA63tBK/93TUtsKfzcIZv3wKXYuvABlEzcSazSxQR7hcLCbWWuI8URnJbhAtvxetqnwNzV7MrdqZUZl88fRs1Qozzc/NSYhgKV15akXDlLM35fcqnuYYMsznReNHy9D85f+izy+fDtVdAQBbrgO5PYsMj9HRMpw7AkLQTaLWgEJZHa+5tU0zbru9wa1+SJrWeZtVOtYnYgNlpTkAACAASURBVIHSTKf75dX7UDKgMGNGWPH0bE1l2WGj2jCxNAbRwnvMjSNrK9H6768AAMfe/QN6/uJ+U8fIxIX1dEAIuga8KdzLFDWp9X4sBxs9MY/etfpSxtNizuy5tGAA14JhRyLWnq3JnkHxnFtO77cCxhiadr6HY++9CKZwtfiaGsHaWkE5uVzHcZm4SQrMIXzoKmilcC9YG53CzYszX4o561JrKt85z/qpfCydgdKpLGsqSWWIp1r5ZBtZ18TCf/oUjqx5Akf/9nS7mNtyUDDuNvxg6gJuMQcQVi5BYC1ihK6ClvDGM9JpaPZqNl8wEsRkTOXNuFkiEc0HAsTjrrGECO3WKTppitMHduHI2v+G78Th0Lacwn7odWUZcnufaepYVpcWEIQjBF2FeIXS7KKTkSDGM5XnqZRo1s2iRPhCw4nVXRMvlRvqdcsGxwLz+3B8y+s4/slKQFFIq8uIiSgY/yvYcjuZPqZamz6BdXAJOhFNBPA0ADuApYyxiojnFwMYF3yYD+AHjDGnlYYmEy0BdToktLT5dYVPrwSq1v5Gghhr5AWvz97IzSLZCWCIEoyCfAnzJlvrwxfwo7xZJ6KrQdMXm3H8f1eEHts6dUGPifcif8iFpo9lI+CG0aK2eaIxFHQisgN4HsDPARwAsJWI1jDGPpf3YYzNVux/D4DiBNiaNLQEdP6VwwBANcpFWV9aq0lBJLyLQ7FO5dXKtqplPRo1YJZvHClzJQiiiGdWxUvncy5G0z834vQ3NcgrGo6ek+5HTreexi9UIJpUJBeeEfooAF8xxvYAABG9DuAqAJ9r7D8dwDxrzEsNPBXvlET+uHgXosyIotmpfFWN27DJgwxPA2bZBkF6EMvitVmIbOg5aQ6aPt+EriVXqTah0EPM4JIPj6C7AOxXPD4AYLTajkQ0AMAgABs1np8FYBYAFBUZJx6kEjMCqvXjMhqpJzLcTy9yJtL3Hk8yjSA1WB1Z1HKwHie2/RU9J80B2dtlwd6lAN1GXWPqWCIsMXXwCLrasrSWSl0P4E3GmOrQgTH2EoCXgEBPUS4L0xQe/6WfMTw1bSTuW1mr+ry70YOBwTrcVseU6/3gI4U65dEZAtOYKdWrB/P7cOLT1Wj8eDng96HReQYK/uOmmI8nsj9TC4+gHwCgXJruB+Cgxr7XA7g7XqPSHV7/pdzHUVmpUItGjxdlq+oAWOPa0PrBF+RLXL1MBemFWobyys/2xxXZ0nbyCI68/SRa9u0IbTv1j3Xodn4p7I5uMR1TLu0sBgWpgUfQtwI4i4gGAXAjINo3RO5EREMAFAD4xFILU4TyB9TdIYEIaGz2oq/TgaaW6PT+SJQui3mTh3HdALx+ZpkbRsuNMm/ysLiPLUguatFKq7e7kZtjg7c1Nj9685ef4OjfnoH/9MnQtty+Q9BzclnMYg4gqaWdBdEYCjpjrI2Ifg1gAwJhiy8zxnYR0SMAtjHG1gR3nQ7gdcYsSk1LEjwp/srFRZ5prhztIvuxS4td2PbtsbCWYVqouUp4YsnV9nXmS8jLseG4xytGTB0YrUS3WPB7T6Nh41Kcqv17+0ayofsFU9H9wuvD/OfxwltHXmAdXJ8eY2w9gPUR2x6OeDzfOrOSg1acdl6OLaYfTGScunKUotYyTI3IBUujWPLImURTaxu8waaQDc1eOCQ7ZowpwqbdhzF7ZS0qN9RHCbuZG0Ys+wv0MbqeVi2Atn6/B0fWVMJ7tD3Gwd61F3pOvh+d+v/YknNEIspCJJeszhS1MsXfIdlBFD1ykkcpPF9syRbd7dyo/ovWTEK5r3JmoHZDMFMwLNYCYwJ1eK6nFQugp/f/E9+tnAv42it25g8Zi8KJ98DeqUtcx9bLjBZlIZJLVhfnsnL0sPCa4WjUqdXC88XuotLpRq+OC28sslYzYUD7hnHfyloUP/JOVHOOWAuMCdpRNkDRah6ivJ7jhvaK+5x5fYZA6hGIbSApDz0uvxc9ryqPS8wlG6EgX9IUcxH6mnyyWtCtGj3Incq1jmcjwrihvaJbhtkJkq09KrSh2YvZK2sxt2qnoY19g31GY8Xd6MHYio26I7+GZi/K3qwLiXpVjVtzfzG15iOykqdWnoJ8Patq3Fjx6X7VfcxAORJ6Tf4t8voNQ5+bn0GXcycYdhPSwyHZ0KVTjm70ltVdmgTGZLWg83RpN0I5CtHrHL9y635MOc8VahlWkC+hzceiws4YAk0jZBHVK8ka7w3J3ehRTTJQ4vUFIm9kIdJCTK354J1V9XU6UFXjRtkq9YbTerC2Vpys24DI+ASpZ3/0vqECUqEVIku6Yi4PcgTJJasFXe7LaAZ5mkkIfGmVoxD5eDYVlfT6GNbtOIQt5eOxeNpInPb6NaeqctMI5THlG4HynGWXDTEUZCMY1DPHlBi5d8TUmh+emYx8Peev2WU6zrz1yD4cevV+HPv7szhV+7eo5+MZlcsYNc0Q34fUkdWLokBAMM3UAff6GfJzc1Dz8ATNfbR+g/KIhmeUpvzhayX9lBa7NLNQzcCgX6bAyL0jptb86HUW8jMWynkw+7kyxnCq7u9oeH8pWFsLAKBh41J0GvQTSM4zLLEdMK4mKtL+U0tWj9CBgI+yqSW6V6ceWuJm5JYAgEHl67huHrwuDJcFrg6X04FFU0eE+fNlJDvpunfE1NocWi60RVNHYPG0kWhqbTPMKo7E5zmBw395DMc2PB8Sc9glFIy7FTnde1tlOgCEZotqyGn/4vuQOrJ6hB5rCVItceMZefNMoNXCF2XUUsBXb3eHnddMgw15eiz/COev2RUKf4yslicKeMWPsm6OsuRy5YZ6NDa3hnIIePF8W4ejby+C79Sx0Dap5wD0vLIMub0GWmk6AIjvQpqTFYKulbgRSwlSvS+uZZEeGm5OtZjl5dX7MPgHnbHncDN8jMFOhOmj++PR0uGqNyzJTuicm6OaPapXz0UU8LIGZas/QnuUi9k4c+ZrQ+PHy3CiejWUt++uP/kFnJfcApuUZ6HVAeSRufgupC+Uqkz9kpIStm3btoSfR2sU7nRImvXC1SDA8ItrFAZohoJ8KcpPz3t8h2QP+bVFVmdySHSrPyVtJ77H4aqFaD30r9A2m6MbelxxH/IHj4rr2EYIH3nqIaLtjLEStecy3oeuNQpv9Hi5I0RcTgf2VkzCuKG9cP8bdRhYvg5nPrA+LF4cMB8GKUeuqNHQ7I1K6uGdAXi8Ptz/Rh0Gla/D/DW70NyqvkagTHAZW7Ex6nwCPiJjy+VsT56krFig3Hz4TjWGHncaWIw+tz6XcDEHAu+tbFWd+K6kKRkv6HoiyBOyJ7tY5lbtxLLqfaEpso8xLKveFybqyhBDI+TjdndImvtEZl+aifX2MQaGwI2rodkbJTS8IiQwhid7Vi8pyyz2Tl3Qc/L9oJxcFIy7DT+YugA5XQotOTYPXj/Dg2/tMN5RkHQyXtCNRJChfaTsdEjIl9ovSUG+FHJdvPbpPtXXL6veFzbClePD1Ubq+ZItLJYcAJo0Rs9A9M3IirhzWWgSkcKfrSN+vfIMAF/0kx7exn9HbevU/8dw3fEyuo26GkTJ/xk3e/1Z8/l2JDJe0I3cILJPsHvQp97s9YeeO634Wy+/I3KEqzW1Luich70Vk0KhXZUb6nWjGtRuRjkWfGIHGz2GImSWbB7x65VnAGJ3tTC/D40fL8fBl2ah+atPo563d3aaPqaViPo96UfGC7rsBinIj3ZtOCQ7xg3thQfe2qlZqdDMl9aosqJyxGa0wCnbJo94ix95B3NW1kJxjwkx9szC0CzDzpEJ6MyXDEXILNlctEureJa8PRZXS9vx7/Dda+U4vmUFwPw4uv5ptJ08GpedViPq96QfGS/oMvm5gQhNWfBkt8em3Yd1R0/yl1bivFLuYF1yNeT6HPJIVgs7Eaac58Lq7e7QiLeh2QsVLQcAfHPUgy3l47G3YhIWTR1huDB72uvTrRETC1aP+DsSm3Yf1txeVeM27SZr+vxDHHz5HrS4vwhtk3oWgT+7ID5kt6BTZ30HEPV70pGMj0OPDBXzMRaWTDPbIMVa/tJ26SRxZ/CpjfblcxpNv+XY5BWf7ucuyhRZJgCAbjkDj9dveSyxVkp7Jv/olTHlasg1cHhl2N/SjGPvvYSmf77XvpFscF40A93GXAuyxVdIjgenQ0LtvEC47KBgA3M1RCJRepLxgq7nCpBL3uqNluVps1atc146BYf4eiNWZYanmQp7kaIpJwgN1PlBKvezAq0eppn6o59btdOwpaCNiNvd0nLoSxxZW4m2hkOhbTnde6Pn5DLkuYbGaS0/Ta1tocV9vbozon5PepLxLhcjV4DRoqk8nXaq+ODN0NDsRdmqOs04STtRzBNqubZ55AKk2roBEIi2sToaRa8qZKZRVePm6g/Lc1Nmfh+OV6/Cv5eVhYl552Hj0OeWZ5Mq5kB7uWRAv+5MJn6umUDGZ4pqLT7KhYSAwA9Ur7odBf+TqEsl2Ym7hodDssGjtjKK9hG+HLkDAHPeqI2K0Ik8nzKzVGCMlRnBbSeP4OAf7wZraQIAUK4DhRPuQpdh4yw5fiwQgL0VkwCI/rHpiF6maMYLulq6daTwlRa7LP2RJhJXsJStTafcLRAQ6SnnubBy636um4XyBieIRilsVv9imj7/EEfWViK37xD0nFxmabnbWBDfhfRGT9Az3oeutkiobJhc9mZdqMKgmSqFqYDQHgJnNJ33eH0xL6wKwrGqBgsAMOaPSgTqfM7FgD0H+YNHg+zW/yRjqb4p6Jhw+dCJaCIR1RPRV0RUrrHPVCL6nIh2EdFr1poZH3rdfbw+FopK4SkFkErM3mzMLKzGu0aQyVhVg6X1+z049Kf/g5ZDX0Y913nI2ISIOcD/vVFmRquRrZnAHQlDQSciO4DnAVwO4BwA04nonIh9zgLwAICxjLFhAO5LgK1xwRs+xtCeop9NnDrdJn6gGsQ7e2GM4cS2v+LQK3Pg/X4vjqyphL+l2SLr4ocAzBxThJqHJ+iKebZmAnckeEboowB8xRjbwxhrBfA6gKsi9rkdwPOMsQYAYIx9b62Z8WPmR9ms0++Th3jbNqbiZuL1s6zI6uRFORq1aXygBfmSYRKXr6kB3785Hw3vLwF8bcFtx9D63VeW2xwrDNrJUTLZnAnckeCZ47kA7Fc8PgBgdMQ+ZwMAEW0BYAcwnzH298gDEdEsALMAoKioKBZ7ddFbkTeKN7cSxgC7jeAz2eAXCHQrmjaqPzbtPsy1+Gklwo8eQC0ZLRKHZMe8ycMAqEcSAYBnz3YcWbcY/ub2Ure5vc8MLHz26JcY42PE6LPP5kzgjgTPCF1teBL59c0BcBaASwBMB7CUiKIqBzHGXmKMlTDGSnr1Uq9/EStGU0KztcrjJRYxBxDq8i738UyWmAPCjy6j5TO3E0XF2JcWu6LEnLV5cez9Jfh+1bwwMe826hqcMfO/007MAeOMXqtr/wgSA88I/QCA/orH/QAcVNmnmjHmBbCXiOoREPitlljJgVFGaGSquzOfP5U/2Syr3odl1erlehOJ7EfP9jhjrVGnn7FQfLaMXKtF1nTvkf04vPYJeL/fG9rH3rkAPSbNgWNQcYIsjg+eyJZsywTuqPAI+lYAZxHRIABuANcDuCFinyoERuZ/IqKeCLhg9lhpqBFaP0J3oweDyteFXDDK+NriR97RFPXOuXY0tcYf2dCRkP3o2SjoSnedlptLbTSqXGz3tzTh0LLfhJKEAMBx5vnoccV9sOd3T5TpMcPTVlFG9BHtGBgKOmOsjYh+DWADAv7xlxlju4joEQDbGGNrgs9NIKLPAfgAlDHGklrrU89HrnTBAO1fznmTh6FsVV3IzSFjI+DqnwSqHUaOSH5S1B1bvj6GTCUbfaK8PnO10ajyetnyOqP7BdPQ+MHLgF1C4fjb0KV4EijeVXIO7MGbEG/MeSzJQ1bW/hEkBq7AV8bYegDrI7Y9rPibAZgT/JcS1KaEkci9NoF2UVf78fpZYNV/4TXDw0Yk44b2wurtmR2mlY0+UT2fuZ8x1dGoPKKP/PZ0G1UK34nv0GXk5cjtNTCxhiMwyl48bWTINmUFSC1xF66SzCWjUv+NypnK8KbFO4NdjIBAiBpj6qVx0xmHZOdOisnWmi6DytepCp+ypomSqho3ylfV4N8fvobOwy9Ny1R9rVIWdiJRXKuDo5f6n1HVFkuLXdhSPt4wjtvj9eG1T/cZ1jhRindDs7fDiblc5lTZM1Wyh18d+VEmV0c0QmtWYiNSzYp8ZPlG7P3THBz/3xU4srYSzNfeF5anY5TVqLnJ9BZ2s/EzzhYyspYLT8x5jFGFHQpf8MernI7LdWuAwKxj3uRhWf8D13LXye44ef2FMYYTO99H7bN3grUGvl+tB+vRtGsjupw7AYSAYCYbtRtSNjYcEWSooPP409MZG6DZbs4MrmDLOy2f6mmNMrzZRmQEB1H0Db/p5HHMnDkDzV9sbt9oy0HBxTeh8/CfAQhcW3sSE8Fk1HqaijDD7CQjBT2ywmKkkDkkO9p8PtWGy+mAFWZJdgo1wJZ/1JEyo4zTz3bkmYxabfzTB77AkbWV8J1or2iRU+gKdBM6Y3DYvskWc0A9bV+EGWYnGSnoAKJcDZHRKiu37ke8xXJTMRpTgwhw5NjQHLxDya6UBWt3Gc5SsjFMUQ9lbRLm9+H4J2/g+JYVAGu/zXY5dwIKLr0dttz0cF9ofYYizDD7yFhBVxL5xR5bsZG7Q5AWcqRMKjI6I5kxugiPlg4P21ZV4+bKhBU+VfXmFcznxXevz0XLgV2h/Wx5nVE48R50HnpRSuzUGkCIz1AgkxWCHkm8o1Jlp6N0EPR1Ow6FinnJU2ueKnjCp6rdvILsEnJ7/zAk6Hn9hqHn5PuR0+0Hlp1bO07cBoCi/N9TzlNPdsv2z1DQTlYKejyVFyN7kaYDDc3e0GhcjsgwcrUQgCnniSm5XvOKgktuQcuBz+E4+wJ0H3MdyGZtcbcLzyzEZ3sbwjKVJRth4TXnhmyL9H+XDCgUfnGBJh1a0GNtYFt22RDdptBaRI6G5q/ZpbN36vB4fYb+fZ4a2NmAPFtr+fdXyOn+A9gd3ULPUU4uzrhxUcI6CW35+hjGnlmIb456VL/Dat9l4RcX6NFhBT1yqqxWq0WL0mIXHvrLTlPFt1wRP7a5VTvTOtHIx5hhlqhYEAX6dMvDF+8uR+PmV+EYPAq9Sh8EEYU+b6OCXfFSvacBXy+8wvLjCrKTDivoRuVyjWjmFHO1dPiqGjeWp4HvXA+HZEMnyaYr6N0d2V3//ODBgzi0ci4ad3wKAPB8+QlO1W1A15ETMW5or7DR8NyqnVhevc/yJuLpECUlyBw6rKCb7aAS6Z7RqodeEGzyID+Xl2OLOs79b9RZ/sO2Go/XD49BoP2J017L65/H6gZLNn/9619x22234ejR9qKguX2GoNOAEQDC3VFVNW6s3u5OyGeeilIBgsylwwq61sKmjShKpNTcM5KNINkpLHzRIdkx6dw+YRUVGz1ezF5Zi/tW1sLpkNDU2pYxoyo/A+5/ow6zV9ZaIr7xuMGSRXNzM37zm9/gD3/4g2IrodsFU+EcOz3kL1cODPQWTuNl+uj+xjsJBJx02OJcWi3lfIxFdSOfvyY6wcbrZ+icmxMqXCUXp9q0+3DUvrJ8N3q8ccevpxs+xizr4p7ujYR37NiB888/P0zMc7v3Qu/pj6PgP24MW/zsGyyboFW10ApmjonOHxAI4qHDCnppsQsLrxmuOmVVikhVjVtz8fK4x4st5eOxt2IStpSPR2mxKyMWCmOdxMcrvunaSJgxhmeeeQajRo3C559/Htp+zTXXYGnVRhScOTJsf4dkD5VN4BVzl9OBbyomYeaYIi43Sl6OTYi5wHI6rKADCDboVR8xyyKiJ1BaVeo6MgX5EmaMKYpZ1N2NnphH6encSPiDDz5AS0sLACA/Px9LlizBm2++iRsv+XFYiWG9mZoWynDWR0uH4+uFV+AblTrqSlra/JhbtTOu9yQQRNJhfegyRmVC9UaHza1tYf1GS4tdHb5S4wlPG0oGFBpmsBIBWksBSr+32iInoJ70kq4V/ogIS5YswWeffYZevXphxYoVGDp0aOh5tdju2Tp5CnKzk+Mer+7aQ4FBI/IVn+4Xo3SBpXR4QTcSEb2s0MjsShmjcL90xscYyt6s0+0t6XRImH/lMJS9Wae6JqB0vUQucpatqgMIodepLXymOsqlpaUFjDF06tQptK1Hjx54//33MXDgQOTl5RkeQ+t7Y6YX56Rz++jeWDNlcV2QPmRECzq9UDm1Wh1aYud0SGhp83dYMbcSuSO8GR+y2abDieCLL77A9OnTcfHFF+Ppp5+O+Thq3xuzLfqMFlTtRCKpSGAavRZ0HX6EDuinQ6uNGrV+ZOmc+ZlszNa7SYeFzyVLluC+++6Dx+NBXV0dJkyYgEmT9H3ZWlgx2zC6JiJkUWA1GSHoRqiVz01UKFqmcKypxdT+8S58xpOQdPToUdx+++34y1/+EtqWl5eH7777Li6b4kXvpihCFgWJgCvKhYgmElE9EX1FROUqz99MRIeJqDb471fWm2odWjHsgnaMskyVxLvwKbs33MF65GZi4jdt2oQRI0aEifmwYcOwdetW3HrrrSmxSUbte+aQ7Hhq2kgh5oKEYCjoRGQH8DyAywGcA2A6EZ2jsutKxtjI4L+lFttpKXox7NlIPNdBDvOLZ+EzloQkr9eLBx98EJdeeinc7naRvfvuu7F161YMHx6fYFqRJCV/zyJDItMla1aQefC4XEYB+IoxtgcAiOh1AFcB+Fz3VWlOabFLNzQtW5BshMrrRmD2ylpTtUrMLhDqYTYh6euvv8YNN9yAzz77LLStR48eePnll3HllVfGbU8sNmkhyt0KkgmPoLsA7Fc8PgBgtMp+U4joPwB8CWA2Y2x/5A5ENAvALAAoKioyb20cqPlotXyceiF/mUbnvMBXwMwiqEOyWTrS1CqUpuWXv+uuu8LE/NJLL8Urr7yCvn37WmKPfG69/AaBIB3h8aGrzccj9W4tgIGMsXMBvAfgz2oHYoy9xBgrYYyV9OrVy5ylcaDlDx03tFeUjzObxBwIRPbI14KXws55lol5VY0bp063RW2X7KTpl3/hhRfQtWtX5OTk4IknnsA777xjqZgD2v7vVCdJCQR68IzQDwBQxlf1A3BQuQNj7Kji4RIA/xW/adah5Q/dtPswFl4znCukMZPxeH14u+4Q9/5WhihWbqgPa8Em0zk3R/OmMWjQICxbtgx9+/ZFSYlqOG7cWJUk1VHKCQsyAx5B3wrgLCIaBMAN4HoANyh3IKI+jDFZEa4E8IWlVsaJnj+0tNiFbd8ew4pP92elmMuYicG30u2g9dkc93jR1taGxx9/HF26dMGcOXPCnrfKV65HvP7vjlBOWJBZGAo6Y6yNiH4NYAMAO4CXGWO7iOgRANsYY2sA3EtEVwJoA3AMwM0JtNk0ev7QuVU7DeueqGG3EWxgMBHdlxFY7XbQ+mx64ATGjRuHjz/+GJIk4eKLL8Z5551n2XmTQbxdtQQCs3DFoTPG1jPGzmaMnckYeyy47eGgmIMx9gBjbBhjbARjbBxjbHcijTaLnj90xadRa7dhaIX0+fzZIeaSneB0SAkLu1P7bLz/2oIvnr8DH3/8ceCx14vnnnvOsnMmi3QtJyzIXLImUxRQ94fepxO6+NS0kViwdpduxbxMwEaB7kWR2IlQee2IhI4mlZ/Nge+P4fRHf8T32/7eboPdjnnz5uHBBx9MmA2JQkTKCJJNRhTnMotyoUrv3Tske1YU6nJItqjMUCvjzHnYvn07pk+fjn/961+hbQMHDsTy5ctx4YUXJsUGq7GiwJdAEIleca4O3eAiFiJDGLUgICvE3IboNP+CfClpouP3+1FZWYkLLrggTMynT5+O2traDivmgMgUFSSfrHC5KOFt+JvpsegupwNNLW2q0S35GiGDiQjBu+OOO7BkyZLQ4y5duuD3v/89Zs6cCcqA0gwiU1SQTLJuhM67IGVU38RuI8wcUwRXB/OHFuRL+KZiEsouG6IZqqh2japq3ChbVReWnFW2qg5VNe5QM+VB5eswtmKjqQJWt99+O3JyAuOKUaNGoba2FjfeeGNGiLlAkGyyboTOkzwk2QjTRvXH6u1uzdH89FH9wyrmDSpflxajeqdD0o0pnzd5mGGoptqi3fw1u6ISgLx+hgfe2gGAYo61Pv/88/H444+jsbER8+fPhyRJhq8RCATqZN0Inat0LgElAwox5TxtQVpWvS9sNNrdkR5C1NLmx8wxRaolDWaOCdTPMYq7dzd6MLB8HQYqRtxaNwmPN7rDk1ZVwh07dmDdunVR28vKyvDYY48JMRcI4iRro1zmr9mlO5ItyJdwqqVNteem2r6tbX40tabHIqrcANpOBB9jof/r+c31kGykmp6vawOAvRWBbkGMMTz33HMoKytDXl4e6urqMHDgQFPHEwgEAUSUSwSlxS7UzpuAp6aN1NynodnLJebyvuki5kBAzIH2JsTy/92Nnpja7OmJuU3D1S27bb7//nv84he/wL333ouWlhacOHECs2bNMm2DQCAwJisFXUmil94yfWnvhtHR7h05C3fDhg0499xzsX79+tBzI0eOxDPPPJNsMwWCrCCrBb1yQ73qQibBGiGW7IROUuZeYqdDwqOlw6NirR/5xdn4aNmTmDhxYlhfzzlz5qC6uhpDhw5VPV480TICgSALo1yUaIUwxrqq4HRI6JyXg4ONHjjzJZw63WaqN2c6I9kpzAXlkOyYf+UwAOGx1rt378b06VNQW9teUqF3797485//jMsuu0zz+KIyoUAQP5k7fORAq6aGy+nQjC/vnGuHUyWiRRa4LeXjsbdiEvJzc0wvJOqRyqbWM8cUofLaEYYZj8uXL8d5550XrRov5AAADipJREFUJuZXXHEFduzYoSvmgDU9PAWCbCerR+hllw1RrbUhl4dVe+6xqwNCFhkpI7tW5GxKq2ur5+XYUlaKoGRAIVfGY7du3dDc3AwAyMvLw43/5yHscl6I0U9uNcwsFZUJBYL4yWpB5+lKo3xu3NBeqNxQj9kra9HdIaGptb11WkOzV7dyY7zEEp1iFbz1uydPnoy7774bmzZtwu0PP4UX/+mD5/hpAMYuFFGZUCCIn6yMQ48Ftcp56YwcTmiF10cZUy7j9Xrx7bffYvDgwWHbPZ6AKP/s6U9UBdrldGBL+fio7aIyoUDAh4hDtwDeol5aOB2SZsy21RTkS3hy6kg8OXWkJdE6kaPkr7/+GhdddBHGjRuHhoaGsOccDgccDodpF4qoTCgQxE9Wu1zMEI8v106E2nkTMKg8Ou09XuTmFC4NH/W2b49hefW+mCN3IlvOvfrqq7jrrrtw6tQpAMCsWbPwxhtvRBXTisWFIioTCgTxIUbonMTjy5UzNRPhD/azQEihmphX1bixafdhMGhndKqh1nLu+PHjmDlzJm666aaQmOfk5OD888+HmttOr+2fQCBIDGKEzkFVjRvNigVQsxTkSxhbsRHuRg8I1tda9/oYFqzdBaB9EVeOg5dDJ8340udfOSzs5vDJJ59gxowZ2Lt3b2jb4MGDsWLFCpSUqLryuBacBQKBtYhFUQO0FkOdDgm/GNFHt8QuEBg9g4XXQ0mEqAPWtcyTbITK60Zg8rlnYOHChZg/fz58vvbj3nrrrXj66afRpUuXuM8lEAjMobcoKkboBmgthp483YaSAYUoGVAYijuPrG5YkC+hsdkbJd4MAXeGUU9Ts1gVgeP1Mzz4yiY8uf0lfPTRR6Ht3bt3x0svvYSpU6dach6BQGAtXD50IppIRPVE9BURlevsdy0RMSJSn4d3QLQWQ32MheKqt5SPxzcVk/D1wivwTcUkLJo6AgX5EhpUxFx53FTEWDsdUlgkiRbfffFpmJhfdNFFqKurE2IuEKQxhi4XIrID+BLAzwEcALAVwHTG2OcR+3UFsA5ALoBfM8Z0/SkdxeUi+761sBPBz1go8ejtukNcSUByVEoik5EiUYvrHqgRecMYQ3H9Urz99tuYOms2vnFNwKGTrcIXLhCkmHjj0EcB+Ioxtocx1grgdQBXqez3/wA8AeB0zJamIUYdjnyMhXpsLqvexyXmFNy/ckM9HAmsxijZSTViRUlBfqAuDfOHu2sKO+di6dKleHTpatT0uBQHT7aG3ucDb+0UlRAFgjSEx4fuArBf8fgAgNHKHYioGEB/xtjbRPQbrQMR0SwAswCgqKjIvLUpQBbA+9+oC4Ufxot8FKvrvSixE6Hy2hGGI+n/O+lHmPXb+Tj5xUc444YnQDkSJDth3uRh6NmzJ9b+uys83nA75aJZ8rHl+jUimkUgSC08gq4WwRxSNiKyAVgM4GajAzHGXgLwEhBwufCZmHpkcYo39T9R0S1q+BkzFNxDhw7hD+W34ujG9wAAjR/+CT+ecm+YIBtlfIqytwJB+sAj6AcA9Fc87gfgoOJxVwA/BvBBMFvwDABriOhKIz96R0IZVx3LyDqZYg60JzFpCe7WD9/FS4+V4ciRI6HXDLUdwsbZY5GXlxd2HL2MT72yt0LQBYLkwuPA3QrgLCIaRES5AK4HsEZ+kjF2nDHWkzE2kDE2EEA1gIwSc5nSYpehT10PvagSK5FsFMrIjBRcv7cF7vXP4fHZt4TEnIjwwAMP4OOPPw4Tc8A441OUvRUI0gfDETpjrI2Ifg1gAwA7gJcZY7uI6BEA2xhja/SPkFnEWqSLKD6fuRzbzkOXTjmqLpPWw9/gyJon4D2yL7TN5XLh1Vdfxbhx41SPZZTxKcreCgTpA1diEWNsPYD1Edse1tj3kvjNSl9iHXnGU8aWAFMLso3N7ZE2fZ0OHGhoxsl/vI2GTS8Dvvbnrr76aixZsgQ9evTQPZ5e0SyjJiECgSB5iOJcJkn2yJMAzBhTZMpdo7Sx7LIh8H6xCQ3vvRgSc8rJw50PVWD16tWGYm6EKHsrEKQPIvXfJGoj0kThdEihQlm8DTaU/nMgILhtc+/BLTXrcerAbuT3ORMVzy7FPVMuscxOUfZWIEgPxAjdJMoRaaJpafOrnlceCc8cUxRKDAICN4DK66Jjz68dNRC1H7yN3/72tzi2d5elYi4QCNIHUW0xDgaVr0t4KKJWyzYt6uvr8cILL2DRokWw2cT9WiDINEQLugSRDH867yIsYwxLly7FT37yEzz11FN4+umnE2yZQCBIN4SgxwFPTDoBGHtmYZirROkmMUJ506iqcWNsxUYMKl+HsRUbQ/VUjh07huuuuw633347mpubAQDz5s2L6vcpEAgyG7EoGgdqMdrjhvbCpt2Hdeua8PYWJSC0wKmV8fnP7Z/gxQWzceDAgdDrfvSjH2HFihUoKCiw4F0KBIKOghD0OIklwkMrGScShvCbhjLChfnacGjzq/i/1W8AinWQO+64A4sWLUJ+fr4pmwQCQcdHuFxSQNllQ1QrnkVip/a9lL50b+O/8e/Xfofjn6wMiXlhYSH+8pe/4A9/+IMQc4EgSxGCngJKi12YMabIUNSV2aGyL/30gc9x6H/uQevB+tBz48ePx44dO1BaWpoIcwUCQQdBCHqKeLR0OBZPG6kbz+6KyPh0SHbk9hoIe373wEabHTfd+yDeffdduFwisUcgyHaEoKeQ0mIXtpSPx1PTRupWNJT3XXjNcPTv3QO9JpfB8YMiPPHnv+LPTz8m4s0FAgEAkViUNqg1oZh87hlYv349Jk+eHLW/z+eD3R5bGV+BQNBx0UssElEuaUJktMz+/fsxfvx4bN68GStXrsTUqVPD9hdiLhAIIhFz9TRk9erVGDFiBDZv3gwAmDVrVlicuUAgEKghBD2NaGpqwqxZs3DttdeGsjxtNhvmzJmDM844I8XWCQSCdEe4XNKEmpoaTJ8+HfX17eGIAwYMwPLlyzF27NgUWiYQCDoKYoSeYvx+P5588kmMHj06TMyvv/561NbWCjEXCATciBF6Cvnuu+9w00034Z133glt69KlC55//nnceOONIOLJJxUIBIIAQtBTSFtbG5ShmyUlJVixYgUGDx6cQqsEAkFHRbhcUojL5cIf//hHEBHKy8uxZcsWIeYCgSBmxAg9iRw7dgyFhYVh20pLS/HFF19gyJAhGq8SCAQCPsQIPQkwxvD73/8eRUVF+Oijj6KeF2IuEAisIGWp/0R0GMC3cRyiJ4AjFpmTaIStiaEj2Qp0LHuFrYnBClsHMMZ6qT2RMkGPFyLaplXPIN0QtiaGjmQr0LHsFbYmhkTbKlwuAoFAkCEIQRcIBIIMoSML+kupNsAEwtbE0JFsBTqWvcLWxJBQWzusD10gEAgE4XTkEbpAIBAIFAhBFwgEggwh7QWdiCYSUT0RfUVE5SrPzyGiz4loBxG9T0QDUmFn0BYjW+8gop1EVEtEHxPROamwM2iLrq2K/a4lIkZEKQsL47iuNxPR4eB1rSWiX6XCzqAthteViKYGv7O7iOi1ZNuosMPoui5WXNMviagxFXYq7DGyt4iINhFRTVAPrkiFnUFbjGwdENSrHUT0ARH1s+TEjLG0/QfADuBrAD8EkAugDsA5EfuMA5Af/PtOACvT2NZuir+vBPD3dLU1uF9XAJsBVAMoSVdbAdwM4LlU2BeDrWcBqAFQEHz8g3S1NWL/ewC8nObX9iUAdwb/PgfAN2ls6yoAvwz+PR7Aq1acO91H6KMAfMUY28MYawXwOoCrlDswxjYxxpqDD6sBWHOnMw+PrScUDzsDSNWKtKGtQf4fgCcAnE6mcRHw2poO8Nh6O4DnGWMNAMAY+z7JNsqYva7TAaxIimXq8NjLAHQL/t0dwMEk2qeEx9ZzALwf/HuTyvMxke6C7gKwX/H4QHCbFrcB+FtCLdKGy1YiupuIvkZAKO9Nkm2RGNpKRMUA+jPG3k6mYSrwfgemBKevbxJR/+SYFgWPrWcDOJuIthBRNRFNTJp14XD/toJuzEEANibBLi147J0PYCYRHQCwHoFZRSrgsbUOwJTg31cD6EpEPeI9cboLulqHB9VRLRHNBFACoDKhFmnDZStj7HnG2JkAfgdgbsKtUkfXViKyAVgM4P6kWaQNz3VdC2AgY+xcAO8B+HPCrVKHx9YcBNwulyAw6l1KRM4E26UG928LwPUA3mSM+RJojxE89k4H8CfGWD8AVwB4NfhdTjY8tv4GwMVEVAPgYgBuAG3xnjjdBf0AAOVoqx9UplFE9DMADwG4kjHWkiTbIuGyVcHrAEoTapE2RrZ2BfBjAB8Q0TcAxgBYk6KFUcPryhg7qvjclwA4L0m2RcLzHTgA4K+MMS9jbC+AegQEPtmY+b5ej9S6WwA+e28D8AYAMMY+AdAJgWJYyYbnO3uQMXYNY6wYAe0CY+x43GdO1SIH5+JCDoA9CEz35MWFYRH7FCOwAHFWB7D1LMXfkwFsS1dbI/b/AKlbFOW5rn0Uf18NoDqNbZ0I4M/Bv3siMDXvkY62BvcbAuAbBJMQU/WP89r+DcDNwb9/hICIJt1uTlt7ArAF/34MwCOWnDuVHxLnxbkCwJdB0X4ouO0RBEbjQGCK/R2A2uC/NWls69MAdgXt3KQnoqm2NWLflAk653VdGLyudcHrOjSNbSUATwL4HMBOANenq63Bx/MBVKTKRpPX9hwAW4Lfg1oAE9LY1msB/Cu4z1IAeVacV6T+CwQCQYaQ7j50gUAgEHAiBF0gEAgyBCHoAoFAkCEIQRcIBIIMQQi6QCAQZAhC0AUCgSBDEIIuEAgEGcL/B/Hv0BaxEQpbAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(preds_df3[(preds_df3['actual']==1) & (preds_df3['t1']!=preds_df3['t2'])]['t3_attn'], preds_df3[(preds_df3['actual']==1) & (preds_df3['t1']!=preds_df3['t2'])]['t3_cont'])\n",
    "xpoints = ypoints = plt.xlim()\n",
    "plt.plot(xpoints, ypoints, linestyle='--', color='k', lw=3, scalex=False, scaley=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 383,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1a8e1cc110>]"
      ]
     },
     "execution_count": 383,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD5CAYAAAA3Os7hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO29e3wU5dn//753syGbcEiCqLAc61dFzlEqVGoVW8XWYhFUtNbW8wkeK1L8BesjaK1QsaKt1oo+Wk+VKNiIQkUrWiuKFUyQBsGiIhAUkCQgZJNssvfvj81sdjczs7O7s8fc79eLF9nZ2Zl79vCZ677u6yCklCgUCoUi+3GkewAKhUKhsAcl6AqFQpEjKEFXKBSKHEEJukKhUOQIStAVCoUiR1CCrlAoFDlCXrQdhBCPAz8G9kopR+g8L4AHgB8BjcBlUsoPox33iCOOkIMHD455wAqFQtGV2bBhw9dSyj56z0UVdOAvwIPAUwbP/xA4tv3fOODh9v9NGTx4MOvXr7dweoVCoVBoCCG+MHouqstFSvk2UGeyy0+Ap2SAdUCxEKJv7MNUKBQKRSLY4UP3ADtDHu9q39YJIcQ1Qoj1Qoj1+/bts+HUCoVCodCwQ9CFzjbdegJSyiVSyrFSyrF9+ui6gBQKhUIRJ3YI+i5gQMjj/sBuG46rUCgUihiwQ9BXAD8XAcYDB6SUX9pwXIVCoVDEgJWwxeeA04EjhBC7gHmAC0BK+WdgFYGQxW0EwhYvT9ZgFQqFQmFMVEGXUl4c5XkJzLBtRAqFQqGIC5UpqlAoFDmCEnSFQqHIEZSgKxQKRY5gJfVfoVAoFO1UVtWyaPVWdjd46VfsZs6k45lSpptLmXKUoCsUirjJZHFLBpVVtcx9cRNeXxsAtQ1e5r64CSAjrlu5XBQKRVxo4lbb4EXSIW6VVbXpHlrSWLR6a1DMNby+Nhat3pqmEYWjBF2hUMSFkbjdVFHNhIVrslLYK6tqmbBwDUPKV+pew+4Gr+7rjLanGiXoCoUiLsxELButdb0Zx6yKam6r3BTcp1+xW/e1RttTjfKhKxRZTDp92P2K3dSaiLpmrS9avTXhcaXiOvVmHBJ4Zt0OVn70JQ2NPooLXbgcAp+/o/6g2+VkzqTjbR1LvIhAomfqGTt2rFQNLhSK+IlcoINA6VMJeExEzy5x1Du/EW6XkwVTR9p2HivXGSuDy1da2s/lFBTl53HA6zN9/5J1ExJCbJBSjtV7TlnoCkWWYmRRgnH0hZ1RGtr+i1ZvNbXUoWPhMPIcVkQvnuu0Suj5reJrC5y9X7Gb3Q3e4IJost7nWFCCrlBkKdFESE9EzaI04hGaKWUeppR5LFnrkeO1KnpWrnP+ipqoVnIvtwshoKExYFlPHNqH5RtqLc0wImnw+mjw+gzHbff7bBUl6ApFllJc6KK+0We6T6TlHC1KI5rFbPS8FWs9cuHQLErmpopqit0u5p87PKqvHgICe1vlJu6aMjJsrKE3DE2Atffl2XU79DvxxEGkWKcrGkYJukKRYuzyrVpZ/hLt59OObySO/YrduhbzrHZx9RS7Gdzbzbuf1hm6O8ysdb2Fw2ji1uD1cXNFNe58Z/QLBZ5dt4Oxg0pNreRQ7F49DL0es/c5mShBVyhSiJ2+1QNec+scAqIVajnOmXS8odhG81XrCVRoUk3oTWraSR5e2fhl0Cp2CJi/ooZZFdXBm5gVy9sPHG6x5hKJvNZUx4ZL4Ji5q2iTkpI0RcMoQVcokoCRFR4t0zAWy72X2xXmRjAiVNhCrdfI88yqqI7nUoM3pdCbVMUHO8NM4IAoh9/Epp3kiduHbYQVKzmZtLVPm+obfbicgmK3K2o0TDS+/PJLfD4fAwcOjLqvCltUKGwmlnC+UNwuZ0yheWV3vhbVhw7gFAK/lFFFZcLCNSkVQO26rETJxHLMteVnAHBb5SaeWbfDluPaMZ5Y8fv9PPLII5SXlzNu3DhWr16NEMI0bFFliioUNhPNd2tEtNC8yKzLBgtiDgGr0UqtlTmTjsftsuavtoPdDV6mlHlYW34GxW6XLccszHdwzNxVDC5fmXQxt/Jexev22bRpExMmTOCGG27g4MGDvP766zz33HNRX6cEXaGwmWT4bkPdMlq9kXjm1maFpKaUeVgwdSSeFKWx93K7gtdixXVkhf/uPRx0eyQLQcDyXjB1ZNQbUa8Yb1SNjY2Ul5dz4oknsm7duuD2Y489Fo8nurtG+dAVCpux6tuOld0N3rjdOZHHMSI0BHHMHa8l5To0Grw+borTb58uIl0od7xcY7r/4ZbWsCgjM1599VVuuOEGPv/88+A2l8tFeXk5t956KwUFBVGPoQRdobCAlVBDbZ9kiWC/Ynfc7pzI40QSen3FhS6kJKlino0IYOLQPmHborm9fG0yajLRV199xaxZs1i6dGnY9lNPPZVHHnmEE044wfIYlaArFFHQCzWc88JG7ni5hvpGH04hkj7NdzkEjS2tlhZBQ1+D6EhVB/3Qucjri+UcXQkJLN9QGxbrbiWSxmxGtG/fPoYNG0Z9fX1wW0lJCYsWLeLyyy/H4YjNK25pbyHE2UKIrUKIbUKIcp3nBwkh3hBCfCSEeEsI0T+mUSgUGYjm372porqTVezzy6DwxSPm908fY9lX7XY5QMQmtE4hWHTBaBadPxpPsTvM72ulVopCn8g1CCsLyWbJRH369OG8884LPv7Zz37Gli1buPLKK2MWc7BgoQshnMBDwJnALuADIcQKKeXmkN3uBZ6SUj4phDgDWABcGvNoFIoMwQ5fdTSshtd5ff6Yj+2XMijc0fy3mdKcIVswiuuvbfAGQ001rCQT3XPPPXz88cfccccdnHnmmQmNzYrL5WRgm5TyMwAhxFLgJ0CooA8DZrX//SZQmdCoFIo0k2yrNdSv+uaWfXEfx8jdIwmUg9XqoUAgU1Pzi5cUujhnVF/e3LLP9hT4XCfS4g5dSDZba3n99deZN28er7zyCqWlpcHX9+7dm3fffdeWsVkRdA+wM+TxLmBcxD4bgWnAA8B5QA8hRG8p5f7QnYQQ1wDXAJaynhSKdJFsqzX0+PEm1XjaKwaaWfdGkST1jb60Jt04BbRl+J2kpNDFoabWmNL3Q8VdY+/evcyaNYu//vWvAJSXl7NkyZKkjNmKk0bobIv8KH4FnCaEqAJOA2qB1k4vknKJlHKslHJsnz59Ip9WKDKGWOOHY0Wz8iqranV/YNFwu5wM7u1OeyZkPHiK3fz+wjHpHoYp908fQ9XtZ7HoguhrEEb4/X4ee+wxhg4dGhRzgOXLl7N//36TV8aPFQt9FzAg5HF/YHfoDlLK3cBUACFEd2CalPKAXYNUKFJJZVUtB5uSF+nhdAgON7cypHwlDiEsuzwcIlBhsV975cO1n9YlbYzJQrNwp5R5MjYG3SEIW3+Ip/7K5s2bufbaa3nnnXfCtv/0pz/lvvvuo3fv3raMNRIrgv4BcKwQYggBy/si4KehOwghjgDqpJR+YC7wuN0DVShSxfwVNfiT6A5o88ugLzuWCBm/DFiJRpUP04EQ5mV8SwpdFObn6fqUPWkonmUFv4y/xLHX6+Xuu+/md7/7HT5fh1HwrW99i4cffpizzjormUOPLuhSylYhxExgNeAEHpdS1ggh7gTWSylXAKcDC4QQEngbmJHEMSsUcWH1R5rJCTWZJICC6DXZ500ebpiAleprEcApx5Ty4Y4DpgveDgFzXtgY9J1reQdgHjH01ltvcfXVV7Nt27bgtry8PObMmcP//u//4nYnv6SCpcQiKeUqYFXEtttD/l4GLLN3aAqFfVitQ25UuErRmWhzi255Dl0xT3Y4qBES2L7fy4KpI01vKE5B2EIoBB7fVFHNC+t38OzV39F93Y4dO8LE/JRTTuGRRx5hxIgRtl1DNFT5XEWXwKg0bKhLoJfbxeGW1rDMSkVi3D99TJiox1MfJjK2OxEE8PnCc4DAzeXWFz+iMcY4/wnHlOqKupSS73//+3z44Yfcc889XHXVVXElB0XDrHyuSv1XdAmMwhDrG33BDMxEXS0OQVJ979lI6Cyosqo2rvfYzrc0NLooMFOIPWlr7ad1fPzxxzQ1NVFWVhbcLoTgiSeeoFu3bhx99NG2jTkWlKArugSp6F6jxLwzoany8XZEsovQGPJ4E8dkawsH3nue0fct59hjj6Wqqor8/Pzg84MGDbJtvPGg6qErugSpbt6g6KC2wctNFdVpzUh1CsGCqSOB+Dszeb/YyO7HZ3Lg3aX4fD42b97MokWL7B5qQigLXZFU7OpwnyjaOTM19jndiEBhxpycZbhdzqCYx7Mg29Z4gPo3/4/D/1kTtn38+PFMnjzZtnHagbLQFUlD81PWNngttUBLNlPKPFErHLocgpJCFwKCndsjn48nszOTcQrB5wvO4b4Lx+TctQFMO8lj2KDbDCklhzb9g92PXR8m5j179uRPf/oTa9euZdSoUckYctwoC12RNMw63KfDSoeA68XIShPA9JMHcNeUkcFtt1Vu4rn3d9ImJU4hmH7yAMYOKs0pS79NSo6ZuyrpNd3ThVb8LJb6PL79u9i/+kGad/4nbPuFF17I/fffT9++fW0do10oC12RNIx+QOks16r1zXSKzraoJLzyYWVVLcs31AaFrk1Klm8IzC5KCpNb6yXV5KqYQ8f3zawueSj+5ka+fOrmMDF39jySE35xNxUVFRkr5qAEXZFEjH5AVn9Y8aI1phhSvpIJC9cEXTza9lkV1YYCFnqzMZthzJs8PKMWWd0u9VM2orj95mt1YdzRrZCeJ7c3nRAOeo6bxjHXPcLdv/x5ModpC8rl0oVJ9oKlnnvDSsH/RNDLCJ3VnuEXLeUbwm82ZjMM7X2a/fzGjLBuF0wdFZau3tXQK3WrcagpvFGz1jpQQ7a1IpzhUthr3Pm01tXSc9xU3Ecdw+8uHJ02N2EsqNt6FyUVC5aaeyPe8qPxoGdVSwLJIFYWxA43twbfg2gzjCllHn5/4ei0W+qifSyLLhhNcZLL/mYibpeTeZOHs+iC0bqLuj6/DGsb19SeTBRY9HyD2keuwle/G5dTBBfBRZ6LIyb/il6eY/l9log5KAu9y5KqBct4y4/GS6L++QavL5jdOGfS8cxZtjGsFIDLKcJmGFPKPKz/oi64cJoOCvM7bijNrbFnPiabZDbR1uLLte+YUfKS9r3Qvve+ulr2r36I5h0fAVD32sNcdt9f+Pbg3hkRZhsvStC7KJm4YGkHdmSEaje2OZOO75R33uaXzH3xo2CUS1G+k5ZWf1rdLo0tgRtzJjZ7drucTDvJQ8UHO5NSI6dNSmZVVAc/r15ul255Aa1hSe3XB2l4fxkH3quAto4ePL66XVT8cxPfHnx6sNdrNqJcLl2UdC1YJhu7/PNadmOkT9Yvw5s2H25pS7vfWvvMMvFmvGDqSO6aMpKi/OTZjqEuQ1+b/gxFCHj77bfZ+9QvOfDOsx1iLhz0/PZ59LvyYVoLisNcM9mIstC7KOlYsEwWkYu7eQ5Bq0WRTaY7IBVon1llVS2ODLsWpxCs/6KORau3pqTGvNHspM17kG2rnuC0ea+Hbc/veyy9J80k/6hjgtsy8aYYC0rQuyiaXzCb/YWgH9ViFc0dsHxDbca5KqxQlO/kt+d1pLRnkphDwB2S7p6njdveZ/+qB/B7Dwa3OfILKf7epXQv+xHCEb6gne0zVCXoXZhUL1gmg3j9xqGLaWMHlQZvbJkliR10y3PQ0upHEhj7+G+VsH1/ICQz0yzzZOJyChadP9pyxyNHt6IwMS887hRKvn8N7uI+IAjz62frDDUUJeg5jlGseaqKZiX7PPFMkbViTXqNgOOtxJdsmlv9hkWmYhXzaH1AM5lF53eEEFoptFUwYARFI8+kaXs1pWdeR+Gx44BAKGOx20VRN/1+p9mKEvQcxqjt2vov6sLcDEbt2JJ1fjvPYxTVEtnlRnvsifLDnTi0T9rdBEaE1haPd1biz1YlJ5A8FHoThkBDb80/37TzP/hbvBQe8+2w15WecSU4nDjyw90pB7w+quclt2lzqlFRLjmMUaz5c+/vNIxBT8X57TyPXjq32+XkkvEDwxKaFk8fw/aF57C2/AzTm8krG7+Mek63y8n908fws/EDcaS4PGFtgzfuGUSblEiy0zp3OgTzJg/vtL251U+b9xv2//0P7PlrOftXPUCb95uwfRwF3TuJOWS/v1wPZaHnMEbuCCt1TJJ5fjvPY+firpUWacVuF/PPDQjL8g21OVk/PNMoKXQxb/LwTp/pPa9uYV/1P6hf8xj+xgMA+BsbOPDOMxw16QbTcNJc8JfroQQ9hzFyRxiF6tltsRid3+7z2LW4a2XmUNQtjyllHiYsXJOVkTHZgqfYbZrg8+mnn1K1ZA5N26vCtruPHU+vcRew6ILRYTf5iUP78OaWfTnlL9fDkqALIc4GHgCcwGNSyoURzw8EngSK2/cpl1KusnmsCh3MFh2NYs31QvWSYbFkW6y7lZmDtk+2xytnOkbfkZaWFu69915+85vf0NTUFNzu7HEEpT+4lsLjvoOn2J0TEVzxEFXQhRBO4CHgTGAX8IEQYoWUcnPIbrcBz0spHxZCDANWAYOTMF5FCNEWHc3cEaGhesmyWLIt1t1K2QBtdpGKptNdGb3vyNq1a7n22mupqanp2Cgc9DjxxxSf+jMc3Qoz2mBIBVYs9JOBbVLKzwCEEEuBnwChgi6Bnu1/9wJ22zlIhT5WCmwZWSqpsmCyyVIy62YE4bOLaPt2vMYRVipAER295iFer5fzzjuPffs6GpCceOKJXHzzb/jbzoKsMBhSgRVB9wA7Qx7vAsZF7DMfeE0I8T9AEfADvQMJIa4BrgEYOHBgrGNVRJCrBbbSReSMorjQhZSB8DY9sShwOUwFvdjt4kAKUt5zDb1oFrfbzb333ssvfvELioqKuOuuu5g5cyZ5eXn8Kg1jzFSsCLpeYFbkitrFwF+klL8XQnwHeFoIMUJKGWaaSCmXAEsAxo4dq+IDEiRVi45dCSszikhXlx5ul5P55w43zGh0CFSEjA4OEfgM6urqKC0tDXvu0ksv5bPPPuOKK65QBqEBVuLQdwEDQh73p7NL5UrgeQAp5XtAAXCEHQNUGGMUg92VfYjJRGthd1NFta6YO4Xo1MjDqO2ZEnN92lpbWbhwIQMGDOCdd94Je04Iwfz585WYm2DFQv8AOFYIMQSoBS4Cfhqxzw7g+8BfhBAnEBD0fSiSSrYtOmYzVqxyv5R8vvCcsG1aA4xn1+3I2DoxmUJz7cccfP1PzN3zOQDXXnstVVVV5Ofnp3lk2UNUQZdStgohZgKrCYQkPi6lrBFC3Amsl1KuAGYDjwohZhFwx1wmZTbmo2Uf2bTomM1YKQJm5Op6c8u+nBBzAcGY7op/77StDry/6RD1/3ySQ9WvEurNzc/PZ8+ePQwYMMD4xYowLMWht8eUr4rYdnvI35uBCfYOTWEVvVh0SNxyj1ZYK1UFvpKNleuIttBs5urKlUVqCTS2BBpDxBO3k+8UXPjtAcFaOVJKGrf8i/o3HqXtcH1wv8LCQn7zm99w4403kpench9jQb1bWY5eLPpNFdU4HYK2dgsqnqJY0WLcU1F4KxVYvQ6zuPNoBb8yJWa9pNAV1u0+HuobfXEXL5MSxg4KLHQ+tfoD9q3+E02fbwjb58c//jEPPvgggwYNSmicXRVVnCvLMXIFtEVMh2MtihWtsFYqCm+lAqvXYbS4Wex2RZ2ZGL021Zg5QUsKXcGO98nC55fMfn4jzZ+8y9dPzgwT8759+7Js2TJWrFihxDwBlKBnObFM5+3YN1rqe7a5F6xex5QyDwumjuyU9NLg9TH3xU1UVtUankN7rVb9sdjtoig/9QLf4PXpVod0u5ycM6qvfoCyzbRJyYvbnbS29/4UQjBz5kw+/vhjpk2bhhApLl+ZYyhBz3JiiTm3Y9/Q1PdEzqGFAA4pX8mEhWtMBTGZxHIdU8o8FOo0O7YyM9FCGLWu9IdbrBX2slv8tYmbs104tRDLlR99Gda9J5m09ezL0d+7mFGjRvHee+/xxz/+kV69eqXk3LmO8qFnIaGLeL3cndOk9Yg1Pj1aYa1YCm9FLjpOHNonJQ02rBDtOiLHbuQL393gNV1crayqZc4LG2OKDLl/+himlHkYUr7SdL94Gl1r+2trLslASknj1rX4Gw/Q48TwcE7HmJ+w/u4luFzWvr8KayhBzzIiF/G0aXSkTrgcgu4FeTQ06qetRyNajLvVGHi9RUe9mOzIGjSpwuw69MYe2QlJo5fbZbq4umj11pjEXKsYCOaLqm6XMyPL+LYe2EPd63/G++kHiLx8CoaU4SrpF3ze07unEvMkINIVLj527Fi5fv36tJw7nSQa6mfU87Kk0EVhfub1R4ylR6eATok56cRo7JGi7nY5KXA5dCNItM8lligXASxut87BOKlJa7ZhVF7A6OaTTKS/jYMfvMSBtc8ifc3B7YUnnEafc+cAnXu6KmJDCLFBSjlW7zlloacQO0L9jBbx6ht9uv7dVKJ3s4plkTTTatAYjV3rTRp6nbMM3Bb1jb6YQwUlgRnDrIrq4PEXTB1pagjouY1Sbbk3797K/tUP4tv7echWQfeyHzHs3GvY20RGGRu5iBL0FGKl3G00zJoia9vT4ZM2ulkVG8Q+61m5mVaDxui91uumY2Qlx4PeZ7lg6kjDDj5GbiM7x2SGv7mRhref4psPVxL6qbr6DKb3pJl08wzl/fmZM/PKZVSUSwqxI9RPL6ZZb2qd6phwo5uVlFhq4pyJU/BYip/ZFWse72epRdH0a585LFq9lYlD++ByJi8MUErJ4a1r2f3YdXzz4SvBkYu8bhSffhl9f3E/3TxDUxENqWhHWegpxKzcrVXfup41ZhZ5kSqMznXA62Px9DFZWSIgluJnevsebm6N2nQ6FDOfd7TPUm+GVPHvnTFHv8SEv42Gt5+m7VBdcFPBkJMoPet6XMVHB7ddMl5VR0wVStBTiFGI3MShfWLyrUcW5DJavEulT9rsZpXNBcRiGXvkvnqLmWaibSa90T5LvRmSXcWzjBDOPHqfPZM9fy3HUVRM6fevoXDoqWHJQQ460v0VyUe5XFJIZMag5mp4c8u+hNLo462Lbmdyj6rN3hm9z/uS8QNjds2Yxfdrn1+yfOUC2L7wHDzFbnxf7yQyKq5gwAh6n3Mznqv+TNEJ3+uU6emHrCsHkc0oCz3F6Fl8RhESVl0m8dRFt7u4llb3+7n3A9N8pxBMOyl7LXO70Pu8xw4qZfbzGy25Q5xC6K4vWKnPbge93C4OHjzIpmX3882HKznixzdTNOz0sH26j9BfrNXItnIQ2YwS9AzAjlZysbo17Ii4CaWyqpblG2qDItUmJcs31DJ2UGnOinq8OQXaPlYE2S+l7jGt1Ge3gy83/pPe/X9K6zf7Aah74zEKhpyE093D8jEyLRw1l1GCbjPx/MhjSaO3C7uKa2nXq3dDSlf2ZypIdIYTOatyGKTvG4lhssMRWw/uo+4fj+D977qw7flHDkG2NgPWBb0ru91SjRJ0G4n3R56OVnJ2zAqs1CfJ1em2HTOc0FmVngvFzHeeLKS/jW82vELDv55G+pqC2x2FxZR+/2oKdfzkZhS6HDl5Q89UlKDHiZ4lbvQjv6mimkWrt5qKdKio1zZ4mf38Rm6qqI7aPCFe7JgVzF9REzWSIlen23aXD452UzebCdlF81fbqHv1j7Ts+TRse/fRkyg+/XKcBd1jPqbXF09vI0W8KEGPAz1LfFZFtWnYWTRrPfKYodXwkpH1aceswEqMda5Ot+2Y4URitA5SWVXLnGUbk1re9vDHb/P1y/eC7BBgV++BlJ49g4L+w+M+bq7e0DMVJehxoGeJW/mpRXb8iWbdR77Obis92fHhQmRXO7pYSPa6R+gMUOhU07SbgsFjcBR0x+89CE4XxRMupufJ5yGc8VdE7Ophq+lACXocJOIX1izuyB6gyTxnIiRSHTJNhTxTQjLXPSJna6l4H53unpSccRWH/7OG0kk3hJW6jRWBKsKVLpSgx0EiTX+dQsQVbmY0dU20HK/ZcQDTRd5oTYc9OT7dTtYMJ5khidLfxjcfrqT14F5Kz7gq7Lmi4RMpGj4xoTZweoXLFKnDUqaoEOJsIcRWIcQ2IUS5zvOLhRDV7f8+EUI02D/UzMFKIaZit0s3czKe2hpm0Q5zX9xEbYMXSYfgxhoFYXScO16uMc1gnTd5uGHxJzXdjh0t8zNZC58tez7lq6d/Rf0bS/jmg0qadtWEPS+ESEjM1WeefqJa6EIIJ/AQcCawC/hACLFCSrlZ20dKOStk//8BypIw1owhMiIlsj6HILBgWFLooluegwavL2iZW20Xpu1nFuViV3KQ0XGMrETN/RP5PlgZs0KfZGZ++lu8NLzzLN+sXxG26Hnwg8qEFjwztalKV8aKy+VkYJuU8jMAIcRS4CfAZoP9Lwbm2TO8zCUyhlhP3OsbfbgcApdTBCMUool5LFNWu0LnYt0/1P2TzYW3MoXKqlpufr46KQufjdv+Td3rD9N2cF/HRqeLXt+5kF7jzk/o2PMmD1effYZhRdA9wM6Qx7uAcXo7CiEGAUOANQbPXwNcAzBwYO6U1NRETW+6bBSnLUTnxa5Yp6x2hc7FsiYgyN1QxHRQWVXL7Bc22i7mrd/sp/4fj9D4ybth2wsGjaL0rBm4ShMT4pJClxLzDMSKD13PqWb09bsIWCal1J03SimXSCnHSinH9unTx+oY00osFQljsXSlDPjZNUoKXTE3ebCrwmEszRkkuRuKmA5uffEj2mxUcyn9fPPhK+x+7LowMXe4e9L7nFkcOf23CYu52+Vk3uT4XTWK5GHFQt8FDAh53B/YbbDvRcCMRAeVKcSayh+rpRuamNMUR0adXaFzesdpbGnVjWDJ9ciVVFJZVUuj7ZmUgsZP3kW2dHwPi0b+gJLTL8dZ2MuWM2RidylFABFZ37jTDkLkAZ8A3wdqgQ+An0opayL2Ox5YDQyR0Q4KjB07Vq5fvz7ecduOUSq/nkA7hcAvpW56tpWFLaMmB9H853aFKFrBqLaI+jHbQ2VVraX8g3jw1e1qj4oAACAASURBVNWy+/GZ5PU6it6TbqBg4Cjbjq3CEtOPEGKDlHKs3nNRLXQpZasQYiYBsXYCj0spa4QQdwLrpZQr2ne9GFhqRcwzDT1L3OzHZpSWrwmd2Ws9cbaMi0z/rm3wMmfZxuC5I/dNhtWuohjsQStqZgfe7dUUDBgeltHpKvVw1IV30q3fUERe/JmekaiwxMzHUmKRlHIVsCpi2+0Rj+fbN6zUkkgiR2SY4JQyD/NX1OjWOSl2u1hbfkZMLePMijL52iR3vFxj2vYs3lowlVW1YdfR2NJquJ8SfWvYWWCr9Zv91L/xKI1b36H4ez+n13cuDHu+YODIhM8RSrHbxfxzVVRLpqMyRUk8rT7y9Ua5GQeafAwpX0kvtysslBH0rR8rLpxIP7cdsel6ZXHrG32dZgR2dz3KZeyKM5f+Ng5Vv0r9P59EtjQCcODdpRQOPRVXSV87hgoYuxUVmY0SdBJL5ddeDx0WmFE6vOaMavAG4tNLCl00NPoMfzRWZw4TFq4JWshm7hwr1nRlVa1hezRfm2T28xuZVVEdXDjVu3nMfl7fFdSVsSOdv2Xv5+x/9UFavgzv0Vk49Hs4uhUmdOxI/FLy+cJzbD2mIvkoQUe/cl4oWmSHnlhqcdm3VW7i2XU7LFVdhEB8emF+HlW3n2W4j9WZgzYuvaxVjeJCV1RrWrMizZKfQtcPzPZRlnoHlVW1CRkM/pYmDrz7HAf//bewTM+8Ug+9z5pBwSD7Fj01VNnb7KTLCnqktTrtJA8rP/qyk3WtuULWf1HXSbAFcMoxpdzxco1pkSojogl2PDMHSecoGrfLiZREdcXYWRQql9vPxYKWBRov3k/Xs//1h2k7sKdjozOPXuMvoNf4CxB5+TaMMhy1+Jm9WCrOlWvoFaNavqGWeZOHc//0MXiK3QgClvmCqYHFpeUbanXF/MMdB+ISc4huBekl/GjuebN4cNn+fOg1HDBoRhF6U7G7KFSutp+LhbkvfhR3FujhLe+wd9n8MDHvNmAE/S7/I8XfvSQpYq59X7r6jThb6ZIWutnC4dryMyy1/ZLAus/qLVdPdLucMTdDiBY6aBQtoxcrbHQdoTcVq4XDQil2u/imqTWmBsddhcqq2oRasBX+v3HklfantW4XjoIelEy8gqKRPzCtiOhywKILxsQc415S6DJ1/ymygy4p6GZFrYwKbelhVfw8IYlKsYb3mRW/iqVrjpV9YxVzt8vJ/HMDKeDJ7N6TjVRW1TIrRlGV0o8QHZNmkeei99kzObRxNSVnXGUp0/PInm6mlHkMF7b1UKn8uUOXFHQj33Qvd/jCoR0ZUskUtliSf6zsa5b0FIleXLKKRw+gFdyy+v3x+5o48O5SfHu30+f8eWEWeMGAERQMGGH53JqxYlXMSwpdqmpiDhE19T9ZpDL1P3IBdOLQPizfUNvJoixwOeL2h+uhWeagb8Em01cZLUTRSoeiULTZiqp1Hp3ht7/K4RZri8vezzZQ99qfaG33kx8x+VcUDTs97nNr7rZojTIEcMn4gdw1xd4EJEXyMUv9z3lBN6pJMu0kD29u2RcmaLMqqm2xyjW2t8fxxuLrtoNodVjMngdlaSfCJY++x9pP66Lu13aonro1j9H48T/Dthcedwp9zrs1rnMLYPH0MYafsYa6KWc3CdVyyXaMFkBf2fglRd06Ln/9F3U44lgUtIJdjSisEi1b1Oj5myqqLf/YVcp/ZyqraqOKuZR+Dm18jYa3nsDffDi43VHQneLTr6D7qB/Eff7Q0sZTyjy6obZgXMZBkf3kvKAbiWaD1xesU1Lb4OWZdTtsPa8QHaJndItIVhRItBuI2Y3ESvq+SvnvTGVVLbOixJu37NtO3eqHaK79OGx70fCJlEy8EmdRcUJjiAxlfXPLPt3vXn2jj1kV1THdwBXZQc4LeqJp/fEipbE/GpK7WBqtk1G09yRaUpBdvUyzCbMZiXaDM5rcSSlp+NczHHx/Gfg73re84r6UnnUD7iGJt+DV6yRlduPWhqpuxrlFzicWxdKNJ1Y8xW7TBB+zUgKJLoiadVIyuubGltZA4S0LNxIzMUi1Cynd6CWizX1xE7dVbmLCwjXcVFFtmmErhKDt0P4OMXc46fmd6fS94kHbxPyS8QM7fZ+szgC1m7Ei+8l5QZ9S5mHB1JHBzMmSQnvqQ7ucgjmTjmfOpON1e/QZISAseSkejARGE3XtmkNb3EFgqq1ZY4Uu84/eTAyMnsvVRCKjGcmz63ZYnv2VTLwCh7sn3TzD6HvZHyj53qU4XN0SHlux28Xi6WN0o1ViMWZy9Wbc1chZQQ+1YBet3sqcScfz+cJzDKfFMdN+nCllnpgiY+wQPTOXh8aUMk/Yom/kftHGbGbF29XLNFswEju991BKP4c2/YM27zdh253unhz9s0UcdclC8vsMSnhMnmI3908fQ/W8swyNA+3GbsXgyNWbcVcjJwXdzILVazwRDz6/ZP6KQBc+q302rYpetMbUVl0eZvuZpaRH6+geOevJ9fofVsWu5esd7PnrXPavup+Gf/6l0/OuUk9YJmg8OIWIaUZo5TPJ5ZtxVyMnF0WtWLB20OD1UVlVy+De0RdeYwkHjNZqzmhR0yEEQ8pXBhftzBZHzcZrJQ3crCRBrjFn0vGdGn6EIltbOPBuBQfeXw7+QEjgoY2rKRrxAwr6n2DrWCLbH67/oq5TPoWeL93o81ZRLrlFTlroZpapXT50jV//bVPU2GMtgcjsR6NZ5TdVVId1MoKOVnMaRr7RNinDZiQTh/bB5Qy351xOwcShfXAYmHmFLof6ceth8H55t1ez+/EZHHivIijmOJz0HH8B+UcNse30ep9XqB9fby1Fw8hFdv/0MQmv5ygyi5y00M0s04lD+5jGnBe7XTG5ZaKleBtNZyN7dkYjtCRBZF0WvYQor6+NlR992cnR62uThtfvcgrunmp/s4RsxqiDU1vjAerXPMbhmjfDtnfznEDppBnk9xls6ziMSvBGbtYLH1UNv7sOOSnoRpUFtRouZvja/DiE8Q8oVrrldZ4E6fXsjJVQl8eQ8pW6+8RSl8YpBIvOH61+5ISXTY6suCml5PCm16l/8wn8TR0Ln6JbESWnX0b30ZMS9pMnit4MtSu5yLoyOSnoRhaJlY48VosqWaXB6+uUuLFo9daYxVwvzDBaJmos+KVUP3g6Z8F2soA/W8/+v/8hbFvhCd+j9IyrcXYvSerYImvqG5V3VhErXRdLpoQQ4mwhxFYhxDYhRLnBPhcKITYLIWqEEH+1d5ixM6XMw9ryM1g8fQyHm1u5qaI6LRmj0HlBNp6Y3/y8cB9oaCSPHSgRCBDtpu/+1li6DQy4pZy9juLI8+fT59xbki7mTiGCkUXaY63dYNj4khixEi36SpF+olroQggn8BBwJrAL+EAIsUJKuTlkn2OBucAEKWW9EOLIZA04FuxwbdhFqIjHU44gsoWcnf0/I0WgKxfeirzZ+luacOQXBB8LIeg9aQaHNr1Or1MuwuEqiDxEUrh43IDgZxA5g0hFaWNVvyc7sGKhnwxsk1J+JqVsAZYCP4nY52rgISllPYCUcq+9w4yPeFwbySLUAp4z6XhcRmEmBmghiZplZJdlHhlDHi0LNdfRPqe2xgN8vXIxXz19M7It/GbqKvVQctplKRNzIJgJqncj18Q8mRErqQoFViSGFUH3ADtDHu9q3xbKccBxQoi1Qoh1Qoiz9Q4khLhGCLFeCLF+37598Y04BjIlnTmycNKUMg+LLhiNO0r6fSihIYlaXHqsaPcQpxD8bPxAti88p5MIdPUf7q/OOo7mzWvY/dj1HP7PG/i+3sHBf/8trWMKTVwzupEn+7ve1er3ZCtWFkX1TMlIszcPOBY4HegP/EsIMUJK2RD2IimXAEsg0OAi5tHGSDIrLeY7BS1t1i4htE51qDsjppS/ECLj1K3Ss8BF9TzzRsBGP9B0rT+kkq1bt/LAzdfy1T/Dm0601e1CSmnanDlZhLrDKqtq07YQGq2CpyIzsGIi7gIGhDzuD+zW2eclKaVPSvk5sJWAwKeVZKYzWxVzjQkL13Bb5SbmLNsYdGdEqysTmRSUKFZi3o1+oAJy1u3S3NzM/PnzGTVqFP8MEfNBgwbx6weepPScm9Mi5tpCaGh0lN5XRq90rt10tfo92YoVQf8AOFYIMUQIkQ9cBKyI2KcSmAgghDiCgAvmMzsHmknE89vWmmhYta4LXQ7a0uD/N6oeKSEn3C6RkRp3PfoCo0aN4o477qClpQUAp9PJnDlzqKmp4c3G/ikZl160yu8vDM8LMCsSluyFya5WvydbiepykVK2CiFmAqsBJ/C4lLJGCHEnsF5KuaL9ubOEEJuBNmCOlHJ/MgceDS3CxW4iY4GThdfnt7W/KVgrHTylzMNNFfqdd7LdXxoZqfFRxb28W/33sH1OPvlklixZwujRo4HEXE0TjinlgrEDg0lKRhj1uLVak8UZUcMnWSKrkpMyH0uJRVLKVcCqiG23h/wtgZvb/2UEyYhwcQjjphV2kwzb3ErRLQhYX7noL41c8HV2Lw3+3aNHDxYsWMDR4yZzw9+3sfu5lRQnWPfnwx0HuGAspg3II90qZuhlQEPngl2gQgm7KjlZnAuSY01mSARkzAjgZzodbYzIVX9p5Hei17jzySvtT+HxE9iyZQueU6Zw20ubg2scsZRO0EOLDjLzfUe6VcyIdHs4dXx/XSkiSdGZnEz9B+gVY5GtXKGk0MW8ycMTSgzKtWJOzc3NLFq0iJLWQdTldVjlIs9F30vvZcDRfejXrx+Lnlpj+wwsWl/PWN9TKzV8st01poifnBX0NAQlJIWSQlfQUix2u/jx6L5U/HunrjvJ7XIyb/JwW3ydueIvffvtt7n22mvZsmULo8d/j4Lvl9PU2tHco6hHr+DMIxlCqLmp9FxYJYUuJixcE/dNU4USKiLJWZdLQ4LT5UzAU+ym6vaz2L7wHLYvPIf55w5n5Udf6op5SaFLRR2EUFdXx5VXXslpp53Gli1bANi47m2mHbGnU6QGBMJK7faoaW4qPReWyyk41NSaUEZurrrGFPGTsxZ6MpOKUoHWhFojMkJDDyXmgfK2zz77LDfffDOh2cg9evTgt7/9LTfccClOZ4cIWnlf40GvrkqoC+twc2snl6BeLXMzcs01pkicnBX0OZOONwy/ywoizMVoxbgSXcDLBf773/9y/fXX88Ybb4Rtnzp1Kg888AD9+3eOKbezyBkELGS9mVKkC8su/3euuMYU9pCzLpdsx+eXMZfc7aolTVtaWvjtb3/LyJEjw8R8wIABvPTSSyxfvjxMzEOTi+ycxcUSgmjk51b+b0Ui5Kyg50LoVm2DNyg8DgurvF2tMqLG+vXrue2222hubgbA4XAwa9YsNm/ezLnnnhu2b2Q1SbvQy+w0Q/m/FckgZwU91qmrp9jN9oXnJGk08SEgKDyRPS2N6IpxyKeccgpXXHEFACeddBIffPAB9913H927d++0r90uFogvDV6l0iuSQdb70I2aMcSyKJqJlpFRVT0Np05jaI1cjkOWUrJz504GDhwYtv2ee+7hxBNP5Lrrrgtb9IwkGe/N2vIz4nqd8n8r7CarLXSzZgxGRaYiCbWMbqvclOwhW8JT7DYVc0+xm08X/CisTnYoueqH/fTTT5k0aRLf/va3qaurC3uud+/ezJgxw1TMwfi9KXa7DN9PhSJbyGpBN2vGMKXMYyqKbpeT+6ePCTZ4qKyq5dl1O5I74CgICI7JTFy0mYeeH1Z7fnD5SgaXr6Tsztey3qfe0tLCggULGDFiBK+//jp79+6lvFy3tW1UjLpFHW5pZeLQPrrvp0KRLWS1oEfromIkinrRCEb1NlLJJSH1VsxcQFoNj1A/rBH1jT7mLNuYsKinq0Hw2rVrOfHEE7n11ltpamoCAoueRUVFSIvrCqFMKfPQvaCzp9HXJnlzyz4WTB0Zb98RhSLtZLWgG02fe7kDVfImDu2j+3xow12NTEhCWvnRl0DHuoARob7zKWWeqBa9r00mtFCajj6j9fX1XHvttXz3u9+lpqYmuL2srIz333+fxYsXx910wiiLeHeDlyllnpiqLCo3jSKTyGpBN5s+V1bV8uYW/b6l2vZQqzMTqG/0cVvlpqB4GqEnItFuSIksBqayz6iUkqVLl3LCCSewZMmS4PaioiLuu+8+/v3vfzN27NiEzhEtBtxq2QiXQ2TcYrqia5PVgm42fTZrKlDb4E1aPHKiPPf+zqhhdZEzD63XpBmJLJSmskHwz3/+cy6++GL27NkT3DZ58mQ2b97MrFmzyMtLPDArWgx4YX50P7oAFl1gPe5coUgFWS3oYD591qsXDQEfdDLike3ASrx55Mwjmv8/si5MrKQyq/HMM8/sOH6/fixfvpyXXnqpU5hiIpjFgFdW1XK4Jfr3YvH0MUrMFRmHiGdhyQ7Gjh0r169fn/BxJixco2uJG3XdyRW2Lzwn6Gs3u858p8Dnl0gZKCnsznPg9fljKuSkV8DKqGZJokgpmTRpEkOHDuWuu+6iZ8+eth4/Gkbfp1CK8p3U3Hl2ikakUIQjhNggpdT1O2a9hW42fc7VBSunEGEuIyOK8p20tAXEHEBKaGzvVRrLwmYyshobGhq44YYbeOedd8K2CyFYtWoVf/jDH1Iu5mDNjXTeicoyV2QmWZ8pGq2EaDJKo6abNimjuoxcTkFjFNdBZLlWo6xbsC+rUUrJCy+8wC9/+Uu++uor3n77bT788EPy8/OD+9jhJ48XKxnGRovtCkW6yXpBB2Ox0bbNfn6j5Voo2YCn2B3VksxzCHxt0a9ZWyCG8JtfMhoOb9++nRkzZrBqVUe/8ZqaGl588UUuuugiW86RKHMmHc+cZRtN37tcLq2gyG6y3uUSjSllHvxxiLnmWsg0NHdStAVJr89v+nwoc1/cxB0v18QVmmgl4cjn87Fo0SKGDRsWJuZ9+/Zl2bJlTJ8+3fJYk82UMg+Lzh9Nocv4p5GrpRUU2Y8lQRdCnC2E2CqE2CaE6JRzLYS4TAixTwhR3f7vKvuHGh+VVbWWSs9GovmZMylrMNR3bZT2Hw9eX5thgwwza9RKwtH777/P2LFjueWWW/B6A8cSQjBjxgw+/vhjpk2bFneCUDKRBp98JhZyUyg0orpchBBO4CHgTGAX8IEQYoWUcnPErhVSyplJGGPcaIKTiLslUxw1JYWusKp+mhvkjpdrktqtyMwaNUs4mvit7vz617/mT3/6U1iK/qhRo1iyZAnjxo1L2pgTxWh9IpYGFgpFOrBioZ8MbJNSfialbAGWAj9J7rDsweiHmYEGIc4oY6pv9HUqtDWlzEPV7WdREkOquhHFblfMDRfMEo727NnDY489FhRzt9vNPffcw/r16zNazMH4uvxSKjFXZDRWBN0D7Ax5vKt9WyTThBAfCSGWCSEG6B1ICHGNEGK9EGJ9aAPfZGH0w5QyIGCZhF+CidsWCIj6TRXVwUqK35q7ktsqN3HOqL4JndvtcjL/3OExhyYalqItdHHcccfx61//GoAf/vCHbN68mTlz5uByZdb7rodqD6fIVqxEuejZjpGeiJeB56SUzUKI64AngU5V/6WUS4AlEEgsinGsMWMWgna4pRWXI5B0kwlIIIZ1TCBwE3hm3Y6EVrYju9PHYoFqESEtvlZavtpGt34Ba/5QU6CWzi233MLo0aOZPHlyRvrJjZgz6XjdRCrlO1dkOla0YBcQanH3B3aH7iCl3C+lbG5/+Chwkj3DSwyzhUNfm6R7QV5GRrLESoz3AQAKXY6wevDxMKXMg9j3KV8+OYs9z83FVx/4WmgNrrt168a5556bVWIOncsSO4UIrg1ke215RW5jRdA/AI4VQgwRQuQDFwErQncQQoTO+c8FPrZviObcVrmJY+auYnD5So6Zuyqs65D2wzSiodFnubNRrtHo8ydUAvfgwYPceOON/PfRX+Lb+xmytYW61x4O+syzPVY7NJJIW1Tvqk24FdlDVEGXUrYCM4HVBIT6eSlljRDiTiGE1lL9RiFEjRBiI3AjcFmyBhzKbZWbeGbdjuAPrk1Knlm3o5OoG/nLe7ldGdHYIl3EUwJXSsmLL77ICSecwB//+Ec075vI60bB4NEgA/OFXPA3p7JssEJhB5YyRaWUq4BVEdtuD/l7LjDX3qFF57n3dxpuv2tKh2VuNONvaW2jwZu8kL9sIBZLeseOHcycOZOXX345bHvRMWPp9YPrcBUfDeSOvzmVZYMVCjvI6kxRo/hybbuWxWgUp90Y6ypkAkSLYEkXVizp1tZWFi9ezLBhw8LE/KijjmLp0qU8/fzfGDx4iG2FuzIFFe2iyDayupaLUwhdUXcKwW2Vm3h23Y6Mcaek8N5hGauW9KWXXsrSpUvDtl133XUsWLCA4uJiAM47sX9SxphOVLSLItvIULvRGheP0w13Z/y3SjJKzDMRAUw7yVoFxeuvvz7494gRI1i7di0PP/xwUMxzlWSUDVYokklWW+ian/y593fSJiVOIbh43ADe3LIv58S80OWIy0UkBCA7Jw5IOpeBDTbMqG/EU1IYjE//3ve+x4wZM+jfvz+zZ8/OiuQgu7CrbLBCkQqyvmORHkPKVxoKupGbJtMpKXTR5PPHVNtd6yo0q6Ja9/0QwOcLzwECYn7T//2Dva/9maKhp1I07DQcQK9CFw2Nvpg6HCkUiuSR0x2L9DBatBIE3DHZSH2jjwVTR8ZUt8Xra2P+ihrDKB/tfWpra2PG3N+wY8l1eP+7jro3HqXN+w3+9vPG2uFIoVCkh5wUdKMMUbfLQc3ub9IwosTRNLnq9rO4f/oYy7VoGrw+9KobCAIiPfp//szxI09k9+pHkL4mAPyNDXg//aDTa1QMtkKR2WS1D90IzS0wf0VNWJx5o8+f0lBFO5EEOi/Nqqiml9vFwabE4ufbWrw0/OsZvtjwcjAZCMDVeyClZ8+goP9w3depGGyFInPJGUHX64dZ1C0vpxKHNN9/otfU+N/3qXv9z7R9E7Io6nRRPOFiep58HsJpbP2rGGyFInPJCUHXGllE9sOMtzl0SaGLwvy8qM2Cs422w/Xsf+1PeD95L2x7waAxlE66AVdJP9PXqxhshSKzyQlBN6q5EQ9ul5N5k4czpcxD2Z2vJbUbUMpxOGneWdPxsLAXJWdcRdGw0ynMd9Lql52aIxflO2lsaQuLctGbDanoF4Ui/eSEoNvl1w1tMVZZVcuhplZbjpsp5BX2pPSMq/h65X10H3UWxadfjtPdA4DmVr/u4qlfwuLpY4KCbTQbgthqqSsUCvvJiSgXI7+u0yBer6RQv93a7y8cHRSlRau3Wm5+MeGY0hhGmxr8LV4Ob3kn+NjlFOQJQeHwifS97AF6//DGoJgDumIOnSNbVAVChSJzyQlB1wtTdLucXDxugO72eZOjt1uz4j8XIpDB+e6ndTjSUFTd6JSN2/7N7v+7ga9fWkjTroCLpbVN4vNLhBDkH3VMTOfZ3eANFjozel9U9ItCkX5ywuUSalVH+nXHDioNpLM3eMM6z8yZdDxryzt1yQMCboX2jHlDtMzNxnZrNVXJp1qmq9Y6LjQLtPWb/dT/4xEaP3k3uH/dqw/R9/IHwCRyJRrFha6oi8wq+kWhSD85IehgXHND22bm941c5GtsaTUVc7fLSZOvDW+KYtoFcMn4gWE13jUWrd7KrrpDHKr+O/X/fBLZ0mEpO9w96Tl+Gjji/5jdLidSmi8yq+gXhSIzyAmXSzTM/L7aIl9tgzeY4m4W2eIpdjPtJE/MYu5yhjtIonloQp+XwPINtbpp9+cPaWPPs3Ooe/3PYWJeNPIH9LvqYbqP+H7MPT21vTVX1AGTuHdVgVChyBy6hKCbdZ7RE3sjPMVu1paf0alKoRWmf3tAmM9+sUn6vhCd3T2RC4+HDx/mlltuYc4lP6R59yfB7Xml/Tnq4rs54kc34SzsZXl8oSK+ePoYti88J9hA2sidor0fSswViswgZ1wuZvQrdusu5vUrdltezAt1K8SzALh8Q22YJVtZVWtYNMvIH1/b4GXCwjXMmXQ8Ly6ey9NPP93xpDOPXuMvpNf4CxB5sfvLQ0MTI1GNHhSK7KBLWOh6UTACmDi0j6H1Wex2GUbBxLMAqFU+hI5YbiPXjpmDRPP/j5t6Nd26dQOg57dG0+/yByn+7k/jEnNPsdvUylaNHhSK7CAn66HrodeSzu1yMu0kD8s31HayPs0Eq7Kq1rDGeDTunz4mGHVjRmiUjZR+kBLh6LgpeYrdXFDwEcXFxfQa9QNu/dt/4sqOjXatCoUiszCrh94lXC6Abhcjr6+NZ9btoKTQRbc8Bwe80Rs5aBEx8d4GtWiaaEgCov35J5vZv/pBCv/fOHp958Lg87UNXm5aeFPwsRCC2c9vjKl5h0el7SsUOYUlQRdCnA08ADiBx6SUCw32Ox94Afi2lDJ15rcFzES0vtGH2+U09SNXVtVyx8s1Cdd20UIjo1noRxcKTm14jXefvBf8bfj2fk7h0O8GC2iJ9jFp451S5mFWRbXlcWxv71SkUChyh6g+dCGEE3gI+CEwDLhYCDFMZ78ewI3A+3YP0g6i+b2N0tcrq2oZc8dr3FRRbUnMi/KdCDDMHNVmAHoNOIJj+XQ9Wx66mt/97nfgb09c8vtp3t0xPgmdxmvVt++xuJ+WHTqkfCUTFq5R3YoUigzHyqLoycA2KeVnUsoWYCnwE539fgPcAzTZOD7biCai0NmK1xYvrdYfLyl0UXPn2Xy+8Bzuu3CM7vnqDjczq6KabnmOTqLfeqiOfS/9jr3L5nNg7+7g9m4DRtDv8j/SffhE0/FaiTqxGp2iF5+vWtApFJmNFZeLB9gZ8ngXMC50ByFEGTBASvmKEOJXRgcSQlwDXAMwcODA2EebAKHlAYzcHZEWbiwx6gANERa80PG0awlJDV4fQe6IpgAACfhJREFULofA6YCW1jYOVb8ayPRsPhzc11HQg5KJl1M08gcI0fneGzneKWUebn3xI8OuTFZ85toagd57pM1ilM9dochMrAi6nvMgqFQioDSLgcuiHUhKuQRYAoEoF2tDtA+tPEBkCVjQt1xjjTd3CBG0YAPHN88m9fkleU317HnhtzTv3hL2XNHwiZSccZVhcpCRpW10TgGGtWs09N6XSFQRLoUic7Ei6LuAASGP+wO7Qx73AEYAb7WnmB8NrBBCnJtpC6MaZsW8QrGyeBlKm5TMfXET3fIcli17X1532poOBR/nlfSl9KwZuAcHMkmbW/2djlXsdjH/3OG6lrJZElU0rMxIVBEuhSJzsSLoHwDHCiGGALXARcBPtSellAeAI7THQoi3gF9lqphrGBXzCmXi0D48s25HTMf1+tpictOIPBe9z57Jnorb6DXufHp950JEXj5ul5P55wYaNcfSHSiRrM5o1rfKDlUoMpuogi6lbBVCzARWEwhbfFxKWSOEuBNYL6VckexBpot4araY0XaonkP/+Qc9x50fVjCrYMAI+l/3BM7uJcFtock+sfisrc4+9DCbkaiYdYUi87EUhy6lXAWsith2u8G+pyc+rMzALn+xlH4ObXyNhreewN98mLyefSgadnrYPqFiDom1c7My+9DDyLpXmaQKRXbQZTJF4yFWH7oeLfu2U7f6IZprPw5uq3vjMdzHjsfhKtB9jVEVxmSTiHWvUCjSjxJ0E+LxoWv4fc0ceK+Cg+8vDyYHAeQV96X0rBsMxdzlEEHfeTqI17pXKBTpRwm6CfH60L2fV1H32kO0NnzVsdHhpGf7oqfD1U23xZ1Z9IpCoVBEQwm6CbH60NsON1C/5jEOb34rbHs3zzBKJ80gv88g3C4nBS6HbhmBom55SswVCkXcdIl66PFiFHNd6NJ/2xrWPhcm5o5uRfQ+eyZHXbKQ/D6DKCl0sWDqyE4ZpRoqaUehUCSCEnQT9Oq/uF1Owx6dxadegsPdE4Cew0+n39V/pvvos4Np+03tWZxGNwqVtKNQKBJBCboJRp16Dre0IVtb8Dc3hu3vdPfkyB/dSN/pd1Ly41/hLAoPRdRqoRjdKFTSjkKhSATlQ4+CXtTHtb/7C3WvPUTBwFH0Pvt/wp47auSpptUZdzd4VXigQqFICkrQY2Dfvn3Mnj2bvRWB5syH6r+kaMQZFPQPhBkK4ECUUruaW0WFByoUCrtRLhcLSCl5/PHHGTp0KE8//XRwu+hWRNuh+o79MPeDK7eKQqFIJkrQo7BlyxZOP/10rrzySurq6oLbC4eeiueqP1M09LvBbR6TbkTFbpdKoVcoFElFuVwMaGpqYsGCBSxYsACfr8ONMnjwYH42+06W7emjW9FQ+ccVCkW6UIKuw1dffcVpp53GJ598EtzmdDqZPXs28+bNo7CwkJPaO/voibbyjysUinSgBF2Ho446iv79+wcFfdy4cSxZsoRRo0YF91GirVAoMg3lQ9dBCMHDDz/MkUceyUMPPcTatWvDxFyhUCgykS4v6Fu3buWaa66hpaUlbPtxxx3HF198wQ033IDT2XmRU6FQKDKNLivozc3N3HHHHYwaNYpHH32U3//+9532KSjQL3GrUCgUmUiXFPS33nqL0aNHM3/+/KBlfvfdd9PQ0JDmkSkUCkX8dClB379/P1dccQUTJ05k69atwe0nn3wy//rXvyguLk7j6BQKhSIxuoSgSyl56qmnGDp0KE888URwe48ePXjwwQd59913GTNmTBpHqFAoFImT82GLn3zyCddffz1r1qwJ2z5t2jQeeOABPB4VeqhQKHKDnBf0ioqKMDEfOHAgDz74IJMnT07jqBQKhcJ+ct7lcssttzB06FAcDgezZ8+mpqZGiblCochJhJSRrYpTdGIh9gFfJOHQRwBfJ+G4qSYXriMXrgFy4zpy4RogN64j0WsYJKXso/dE2gQ9WQgh1kspx6Z7HImSC9eRC9cAuXEduXANkBvXkcxryHmXi0KhUHQVlKArFApFjpCLgr4k3QOwiVy4jly4BsiN68iFa4DcuI6kXUPO+dAVCoWiq5KLFrpCoVB0SZSgKxQKRY6QtYIuhDhbCLFVCLFNCFGu8/x1QohNQohqIcQ7Qohh6RinGdGuIWS/84UQUgiRkeFaFj6Ly4QQ+9o/i2ohxFXpGKcZVj4LIcSFQojNQogaIcRfUz1GK1j4LBaHfA6fCCEyssSohesYKIR4UwhRJYT4SAjxo3SM0wwL1zBICPFG+/jfEkL0T/ikUsqs+wc4gU+BbwH5wEZgWMQ+PUP+Phd4Nd3jjvUa2vfrAbwNrAPGpnvccX4WlwEPpnusCV7DsUAVUNL++Mh0jzve71TI/v8DPJ7uccf5eSwBrm//exiwPd3jjuMaXgB+0f73GcDTiZ43Wy30k4FtUsrPpJQtwFLgJ6E7SCkPhjwsAjJt9TfqNbTzG+AeoCmVg4sBq9eRyVi5hquBh6SU9QBSyr0pHqMVYv0sLgaeS8nIYsPKdUigZ/vfvYDdKRyfFaxcwzDgjfa/39R5PmayVdA9wM6Qx7vat4UhhJghhPiUgCDemKKxWSXqNQghyoABUspXUjmwGLH0WQDT2qeWy4QQA1IzNMtYuYbjgOOEEGuFEOuEEGenbHTWsfpZIIQYBAwB1ug9n2asXMd84GdCiF3AKgKzjUzCyjVsBKa1/30e0EMI0TuRk2aroAudbZ0scCnlQ1LKY4D/D7gt6aOKDdNrEEI4gMXA7JSNKD6sfBYvA4OllKOAfwBPJn1UsWHlGvIIuF1OJ2DZPiaEyLSOKJZ+F+1cBCyTUrYlcTzxYuU6Lgb+IqXsD/wIeLr9N5MpWLmGXwGnCSGqgNOAWqA1kZNm0hsQC7uAUCuvP+ZTrqXAlKSOKHaiXUMPYATwlhBiOzAeWJGBC6NRPwsp5X4pZXP7w0eBk1I0NqtY+T7tAl6SUvqklJ8DWwkIfCYRy+/iIjLT3QLWruNK4HkAKeV7QAGBoleZgpXfxW4p5VQpZRnw6/ZtBxI6a7oXD+JccMgDPiMwZdQWHIZH7HNsyN+TgfXpHnes1xCx/1tk5qKolc+ib8jf5wHr0j3uOK7hbODJ9r+PIDCd7p3uscfznQKOB7bTnliYaf8sfh5/By5r//sEAmKZMddj8RqOABztf/8WuDPh86b7whN4w34EfEJgJfnX7dvuBM5t//sBoAaoJrDgYCiWmXoNEftmpKBb/CwWtH8WG9s/i6HpHnMc1yCA+4DNwCbgonSPOd7vFAH/88J0jzXBz2MYsLb9O1UNnJXuMcdxDecD/23f5zGgW6LnVKn/CoVCkSNkqw9doVAoFBEoQVcoFIocQQm6QqFQ5AhK0BUKhSJHUIKuUCgUOYISdIVCocgRlKArFApFjvD/A/PjpnWTwRInAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(preds_df3[(preds_df3['actual']==1) & (preds_df3['t1']!=preds_df3['t2'])]['t3_attn'], preds_df3[(preds_df3['actual']==1) & (preds_df3['t1']!=preds_df3['t2'])]['t3_cont'])\n",
    "xpoints = ypoints = plt.xlim()\n",
    "plt.plot(xpoints, ypoints, linestyle='--', color='k', lw=3, scalex=False, scaley=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Calculate residuals for the DLAW model and 3 naive models: 50/50 split, last touched, and first touched"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 384,
   "metadata": {},
   "outputs": [],
   "source": [
    "#preds_df3['cont_error'] = ( abs(preds_df3['t1_cont']-preds_df3['t1_attn'])+abs(preds_df3['t2_cont']-preds_df3['t2_attn']) ) / (num_time_steps-1)\n",
    "preds_df3['cont_error'] = ( abs(preds_df3['t1_cont']-preds_df3['t1_attn'])+abs(preds_df3['t2_cont']-preds_df3['t2_attn'])+abs(preds_df3['t3_cont']-preds_df3['t3_attn']) ) / num_time_steps\n",
    "preds_df3['5050_cont_error'] = (abs(preds_df3['t1_cont']-(1/num_time_steps)) + abs(preds_df3['t2_cont']-(1/num_time_steps)) +abs(preds_df3['t3_cont']-(1/num_time_steps)) ) / num_time_steps\n",
    "preds_df3['last_touch_cont_error'] = 1-preds_df3['t3_cont']\n",
    "preds_df3['first_touch_cont_error'] = 1-preds_df3['t1_cont']\n",
    "#preds_df3 = preds_df3[preds_df3['t1']!=preds_df3['t2']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 385,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAN6ElEQVR4nO3df4xl5V3H8fenYGms0S7ugGShHTBr4pIo6EgbG5UGI7/SLsbSgFFXJFl/LDFG/9mKSU2TJquJGkyQZk0JS6IFqlY2glVcq03/oO2AyI9WZFlW2O6GnQKhVAxm6dc/5gxehtmdO3N/zJ1n3q/k5p773Ofc883ZPZ955jnnnklVIUlqy9vWugBJ0vAZ7pLUIMNdkhpkuEtSgwx3SWrQ6WtdAMDmzZtrenp6rcuQpHXloYce+kZVTS313kSE+/T0NLOzs2tdhiStK0n+62TvOS0jSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNmohvqI7L9O77+up3eM/VI65EkkbLkbskNchwl6QGrftpmX6nWiRpI3HkLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lq0LJ/rCPJecCdwPcB3wb2VtUtSc4E7gamgcPAR6rqpSQBbgGuAl4FfrmqHh5N+aPh31qVtN71M3I/AfxOVf0g8D5gV5JtwG7gQFVtBQ50rwGuBLZ2j53AbUOvWpJ0SsuGe1UdWxh5V9UrwNeALcB2YF/XbR9wTbe8Hbiz5j0IvCvJOUOvXJJ0Uiuac08yDVwMfAk4u6qOwfwPAOCsrtsW4Lme1Y50bZKkMek73JN8F/DXwG9V1TdP1XWJtlri83YmmU0yOzc3128ZkqQ+9BXuSb6D+WD/i6r6m675+YXplu75eNd+BDivZ/VzgaOLP7Oq9lbVTFXNTE1NrbZ+SdISlg337uqXTwFfq6o/7nlrP7CjW94B3NvT/kuZ9z7g5YXpG0nSeCx7KSTwfuAXgceSPNK1/S6wB7gnyY3As8C13Xv3M38Z5EHmL4W8YagVS5KWtWy4V9UXWXoeHeCyJfoXsGvAuiRJA/AbqpLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBp691ARvB9O77+up3eM/VI65E0kbhyF2SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1aNlwT3J7kuNJHu9p+/0kX0/ySPe4que9jyY5mOTJJJePqnBJ0sn1M3K/A7hiifY/qaqLusf9AEm2AdcBF3br/FmS04ZVrCSpP8uGe1V9AXixz8/bDtxVVa9V1TPAQeCSAeqTJK3CIHPuNyV5tJu22dS1bQGe6+lzpGt7iyQ7k8wmmZ2bmxugDEnSYqsN99uA7wcuAo4Bf9S1Z4m+tdQHVNXeqpqpqpmpqalVliFJWsqq7i1TVc8vLCf5c+DvupdHgPN6up4LHF11dROu33vGSNK4rWrknuScnpc/CyxcSbMfuC7JGUnOB7YCXx6sREnSSi07ck/yaeBSYHOSI8DHgEuTXMT8lMth4FcBquqJJPcAXwVOALuq6vXRlC5JOpllw72qrl+i+VOn6P8J4BODFCVJGozfUJWkBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1KBl/8yexmd693199Tu85+oRV3Jy66FGSY7cJalJjtw1Eo7wpbXlyF2SGmS4S1KDDHdJapBz7tqQ+j0nAJ4X0PpkuAtYWdhJmnyGe8MMbGnjcs5dkhpkuEtSgwx3SWqQc+7rkHPpkpbjyF2SGmS4S1KDDHdJapDhLkkNMtwlqUFeLaM15X3fpdFYduSe5PYkx5M83tN2ZpIHkjzVPW/q2pPkT5McTPJokh8ZZfGSpKX1My1zB3DForbdwIGq2goc6F4DXAls7R47gduGU6YkaSWWDfeq+gLw4qLm7cC+bnkfcE1P+50170HgXUnOGVaxkqT+rPaE6tlVdQygez6ra98CPNfT70jX9hZJdiaZTTI7Nze3yjIkSUsZ9tUyWaKtlupYVXuraqaqZqampoZchiRtbKsN9+cXplu65+Nd+xHgvJ5+5wJHV1+eJGk1Vnsp5H5gB7Cne763p/2mJHcB7wVeXpi+kdYrL9fUerRsuCf5NHApsDnJEeBjzIf6PUluBJ4Fru263w9cBRwEXgVuGEHNkqRlLBvuVXX9Sd66bIm+BewatChJ0mC8/YAkNcjbD2hdcN5bWhlH7pLUIMNdkhpkuEtSg5xzV1P84+HSPEfuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAb5JSZpSLy5mSaJI3dJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSg/wSkzRmftlJ4+DIXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUED3RUyyWHgFeB14ERVzSQ5E7gbmAYOAx+pqpcGK1OStBLDGLl/oKouqqqZ7vVu4EBVbQUOdK8lSWM0immZ7cC+bnkfcM0ItiFJOoVBw72Af0zyUJKdXdvZVXUMoHs+a6kVk+xMMptkdm5ubsAyJEm9Bv1LTO+vqqNJzgIeSPIf/a5YVXuBvQAzMzM1YB2SpB4Djdyr6mj3fBz4LHAJ8HyScwC65+ODFilJWplVj9yTvBN4W1W90i3/DPBxYD+wA9jTPd87jEKljca/tapBDDItczbw2SQLn/OXVfW5JF8B7klyI/AscO3gZUqSVmLV4V5Vh4AfXqL9BeCyQYqSJA3Gb6hKUoMMd0lqkOEuSQ0y3CWpQYN+iUnSBublmpPLcJc0cv4QGD/DXVrnDE4txTl3SWqQ4S5JDTLcJalBzrlLG4Rz8xuLI3dJapDhLkkNMtwlqUGGuyQ1yBOqkt6k3xOvmmyO3CWpQYa7JDXIcJekBhnuktQgw12SGuTVMpImxrBvkbCRb7ngyF2SGmS4S1KDDHdJapDhLkkN8oSqpHXHWyQsz5G7JDXIcJekBhnuktQgw12SGmS4S1KDvFpG0obX4m0KHLlLUoMcuUtSn1Zyff1aj/IduUtSgxy5S9IIrPU8/shG7kmuSPJkkoNJdo9qO5KktxpJuCc5DbgVuBLYBlyfZNsotiVJeqtRjdwvAQ5W1aGq+l/gLmD7iLYlSVpkVHPuW4Dnel4fAd7b2yHJTmBn9/JbSZ5c5bY2A99Y5brjZJ3DZZ3DZZ3D1Xed+YOBtvOek70xqnDPEm31phdVe4G9A28oma2qmUE/Z9Ssc7isc7isc7gmoc5RTcscAc7reX0ucHRE25IkLTKqcP8KsDXJ+UneDlwH7B/RtiRJi4xkWqaqTiS5CfgH4DTg9qp6YhTbYghTO2NincNlncNlncO15nWmqpbvJUlaV7z9gCQ1yHCXpAZNdLgvdwuDJGckubt7/0tJpnve+2jX/mSSyyexziTTSf4nySPd45NrXOdPJnk4yYkkH1703o4kT3WPHRNa4+s9+3KkJ/D7qPO3k3w1yaNJDiR5T897Y9mXQ6hzkvbnryV5rKvli73feJ+wY33JOsd9rANQVRP5YP5E7NPABcDbgX8Hti3q8xvAJ7vl64C7u+VtXf8zgPO7zzltAuucBh6foP05DfwQcCfw4Z72M4FD3fOmbnnTJNXYvfetCdqXHwC+s1v+9Z5/87Hsy0HrnMD9+d09yx8CPtctT9qxfrI6x3asLzwmeeTezy0MtgP7uuW/Ai5Lkq79rqp6raqeAQ52nzdpdY7TsnVW1eGqehT49qJ1LwceqKoXq+ol4AHgigmrcZz6qfPzVfVq9/JB5r/rAePbl4PWOU791PnNnpfv5P+/FDlRx/op6hy7SQ73pW5hsOVkfarqBPAy8L19rjsJdQKcn+Tfkvxrkp8YUY391jmKdVdi0O28I8lskgeTXDPc0t5kpXXeCPz9KtcdxCB1woTtzyS7kjwN/CHwmytZdwLqhPEd68Bk38992VsYnKJPP+sOyyB1HgPeXVUvJPlR4G+TXLjop/+wDLJPxrU/B93Ou6vqaJILgH9O8lhVPT2k2nr1XWeSXwBmgJ9a6bpDMEidMGH7s6puBW5N8vPA7wE7+l13SAapc5zHOjDZI/d+bmHwRp8kpwPfA7zY57prXmf3q+QLAFX1EPPzeT+whnWOYt2VGGg7VXW0ez4E/Atw8TCL69FXnUl+GrgZ+FBVvbaSdSegzonbnz3uAhZ+k5i4/dnjjTrHfKzPG+cE/0oezP9WcYj5kyQLJy8uXNRnF28+UXlPt3whbz7JcojRnWQZpM6phbqYP0nzdeDMtaqzp+8dvPWE6jPMnwDc1C0Pvc4Ba9wEnNEtbwaeYtHJrjH/m1/M/AG8dVH7WPblEOqctP25tWf5g8Bstzxpx/rJ6hzbsf7G9kf54UPYmVcB/9n957u5a/s48yMMgHcAn2H+JMqXgQt61r25W+9J4MpJrBP4OeCJ7j/Jw8AH17jOH2N+dPLfwAvAEz3r/kpX/0HghkmrEfhx4LFuXz4G3LjG+/KfgOeBR7rH/nHvy0HqnMD9eUt3rDwCfJ6eUJ2wY33JOsd9rFeVtx+QpBZN8py7JGmVDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUoP8Dwy6cEfNKTFYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "_=plt.hist(preds_df3['cont_error'], bins=30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 386,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([321., 332., 157.]),\n",
       " array([0.        , 0.66666667, 1.33333333, 2.        ]),\n",
       " <a list of 3 Patch objects>)"
      ]
     },
     "execution_count": 386,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAARYElEQVR4nO3df4xlZX3H8fenLGL9kQLuYLfL4qLdtkJSgU4o1aZFMRUx7WJamyWtrpZmtcVGU9MENam2qSkmVRrTlgaFuDaWH/VHpYqtiBhjDeBAcQFXZEUq627YVRAlprTgt3/cZ+QyO7tzZ+be2fHx/Upu7jnPec4933nm7OeeOefes6kqJEl9+YnDXYAkafwMd0nqkOEuSR0y3CWpQ4a7JHVozeEuAGDt2rW1cePGw12GJP1IueWWW75VVVPzLVsV4b5x40ZmZmYOdxmS9CMlyX8fbJmnZSSpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOr4huq6s/GCz9xuEvQHPde9LLDXYJWkEfuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUM/8h+F9CN3knQgj9wlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDC4Z7kicnuTnJl5LcmeQvWvuJSW5KcneSq5I8qbUf1eZ3teUbJ/sjSJLmGuXI/RHgRVX1POAU4OwkZwDvBC6uqk3Ag8D5rf/5wINV9bPAxa2fJGkFLRjuNfBwmz2yPQp4EfCh1r4dOLdNb27ztOVnJcnYKpYkLWikc+5JjkhyG7APuA74GvCdqnq0ddkNrG/T64H7ANryh4BnzPOa25LMJJnZv3//8n4KSdITjBTuVfVYVZ0CHA+cDjx3vm7teb6j9DqgoerSqpququmpqalR65UkjWBRn5apqu8AnwXOAI5OMnvL4OOBPW16N7ABoC3/KeCBcRQrSRrNKJ+WmUpydJv+SeDFwE7gBuB3WretwMfa9DVtnrb8M1V1wJG7JGlyRvnPOtYB25McweDN4Oqq+niSLwNXJvkr4L+Ay1r/y4B/SrKLwRH7lgnULUk6hAXDvap2AKfO034Pg/Pvc9v/B3jFWKqTJC2J31CVpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdWjDck2xIckOSnUnuTPKG1v72JN9Mclt7nDO0zpuT7EpyV5KXTPIHkCQdaM0IfR4F3lRVtyZ5OnBLkuvasour6m+GOyc5CdgCnAz8DPDpJD9XVY+Ns3BJ0sEteOReVXur6tY2/T1gJ7D+EKtsBq6sqkeq6uvALuD0cRQrSRrNos65J9kInArc1Jpen2RHksuTHNPa1gP3Da22m3neDJJsSzKTZGb//v2LLlySdHAjh3uSpwEfBt5YVd8FLgGeA5wC7AXeNdt1ntXrgIaqS6tquqqmp6amFl24JOngRgr3JEcyCPYPVtVHAKrq/qp6rKp+ALyXx0+97AY2DK1+PLBnfCVLkhYyyqdlAlwG7Kyqdw+1rxvq9nLgjjZ9DbAlyVFJTgQ2ATePr2RJ0kJG+bTMC4BXArcnua21vQU4L8kpDE653Au8FqCq7kxyNfBlBp+0ucBPykjSylow3Kvq88x/Hv3aQ6zzDuAdy6hLkrQMfkNVkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1aMFwT7IhyQ1Jdia5M8kbWvuxSa5Lcnd7Pqa1J8l7kuxKsiPJaZP+ISRJTzTKkfujwJuq6rnAGcAFSU4CLgSur6pNwPVtHuClwKb22AZcMvaqJUmHtGC4V9Xeqrq1TX8P2AmsBzYD21u37cC5bXoz8IEauBE4Osm6sVcuSTqoRZ1zT7IROBW4CXhmVe2FwRsAcFzrth64b2i13a1t7mttSzKTZGb//v2Lr1ySdFAjh3uSpwEfBt5YVd89VNd52uqAhqpLq2q6qqanpqZGLUOSNIKRwj3JkQyC/YNV9ZHWfP/s6Zb2vK+17wY2DK1+PLBnPOVKkkYxyqdlAlwG7Kyqdw8tugbY2qa3Ah8ban9V+9TMGcBDs6dvJEkrY80IfV4AvBK4Pcltre0twEXA1UnOB74BvKItuxY4B9gFfB94zVgrliQtaMFwr6rPM/95dICz5ulfwAXLrEuStAx+Q1WSOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ2sOdwGSVsbGCz9xuEvQPO696GUTed0Fj9yTXJ5kX5I7htrenuSbSW5rj3OGlr05ya4kdyV5yUSqliQd0iinZd4PnD1P+8VVdUp7XAuQ5CRgC3ByW+cfkhwxrmIlSaNZMNyr6nPAAyO+3mbgyqp6pKq+DuwCTl9GfZKkJVjOBdXXJ9nRTtsc09rWA/cN9dnd2g6QZFuSmSQz+/fvX0YZkqS5lhrulwDPAU4B9gLvau2Zp2/N9wJVdWlVTVfV9NTU1BLLkCTNZ0nhXlX3V9VjVfUD4L08fuplN7BhqOvxwJ7llShJWqwlhXuSdUOzLwdmP0lzDbAlyVFJTgQ2ATcvr0RJ0mIt+Dn3JFcAZwJrk+wG3gacmeQUBqdc7gVeC1BVdya5Gvgy8ChwQVU9NpnSJUkHs2C4V9V58zRfdoj+7wDesZyiJEnL4+0HJKlDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShxYM9ySXJ9mX5I6htmOTXJfk7vZ8TGtPkvck2ZVkR5LTJlm8JGl+oxy5vx84e07bhcD1VbUJuL7NA7wU2NQe24BLxlOmJGkxFgz3qvoc8MCc5s3A9ja9HTh3qP0DNXAjcHSSdeMqVpI0mqWec39mVe0FaM/Htfb1wH1D/Xa3tgMk2ZZkJsnM/v37l1iGJGk+476gmnnaar6OVXVpVU1X1fTU1NSYy5CkH29LDff7Z0+3tOd9rX03sGGo3/HAnqWXJ0laiqWG+zXA1ja9FfjYUPur2qdmzgAemj19I0laOWsW6pDkCuBMYG2S3cDbgIuAq5OcD3wDeEXrfi1wDrAL+D7wmgnULElawILhXlXnHWTRWfP0LeCC5RYlSVoev6EqSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6tGY5Kye5F/ge8BjwaFVNJzkWuArYCNwL/G5VPbi8MiVJizGOI/cXVtUpVTXd5i8Erq+qTcD1bV6StIImcVpmM7C9TW8Hzp3ANiRJh7DccC/gU0luSbKttT2zqvYCtOfjlrkNSdIiLeucO/CCqtqT5DjguiRfGXXF9mawDeCEE05YZhmSpGHLOnKvqj3teR/wUeB04P4k6wDa876DrHtpVU1X1fTU1NRyypAkzbHkcE/y1CRPn50GfgO4A7gG2Nq6bQU+ttwiJUmLs5zTMs8EPppk9nX+uar+PckXgauTnA98A3jF8suUJC3GksO9qu4BnjdP+7eBs5ZTlCRpefyGqiR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdmli4Jzk7yV1JdiW5cFLbkSQdaCLhnuQI4O+BlwInAeclOWkS25IkHWhSR+6nA7uq6p6q+l/gSmDzhLYlSZpjzYRedz1w39D8buCXhzsk2QZsa7MPJ7lridtaC3xrietO0mqtC1Zvbda1ONa1OKuyrrxzWXU962ALJhXumaetnjBTdSlw6bI3lMxU1fRyX2fcVmtdsHprs67Fsa7F+XGra1KnZXYDG4bmjwf2TGhbkqQ5JhXuXwQ2JTkxyZOALcA1E9qWJGmOiZyWqapHk7we+A/gCODyqrpzEttiDKd2JmS11gWrtzbrWhzrWpwfq7pSVQv3kiT9SPEbqpLUIcNdkjq0qsN9oVsYJDkqyVVt+U1JNg4te3NrvyvJS1a4rj9N8uUkO5Jcn+RZQ8seS3Jbe4z1IvMIdb06yf6h7f/h0LKtSe5uj60rXNfFQzV9Ncl3hpZNcrwuT7IvyR0HWZ4k72l170hy2tCySY7XQnX9XqtnR5IvJHne0LJ7k9zexmtmhes6M8lDQ7+vPx9aNrHbkYxQ158N1XRH26eObcsmMl5JNiS5IcnOJHcmecM8fSa7f1XVqnwwuBD7NeDZwJOALwEnzenzx8A/tuktwFVt+qTW/yjgxPY6R6xgXS8EntKm/2i2rjb/8GEcr1cDfzfPuscC97TnY9r0MStV15z+f8LgAvxEx6u99q8BpwF3HGT5OcAnGXxv4wzgpkmP14h1PX92ewxu8XHT0LJ7gbWHabzOBD6+3H1g3HXN6fubwGcmPV7AOuC0Nv104Kvz/Huc6P61mo/cR7mFwWZge5v+EHBWkrT2K6vqkar6OrCrvd6K1FVVN1TV99vsjQw+5z9py7nlw0uA66rqgap6ELgOOPsw1XUecMWYtn1IVfU54IFDdNkMfKAGbgSOTrKOyY7XgnVV1RfadmHl9q9RxutgJno7kkXWtSL7V1Xtrapb2/T3gJ0Mvrk/bKL712oO9/luYTB3cH7Yp6oeBR4CnjHiupOsa9j5DN6dZz05yUySG5OcO6aaFlPXb7c/AT+UZPaLZqtivNrpqxOBzww1T2q8RnGw2ic5Xos1d/8q4FNJbsngFh8r7VeSfCnJJ5Oc3NpWxXgleQqDkPzwUPPExyuD08WnAjfNWTTR/WtStx8YhwVvYXCIPqOsu1Qjv3aS3wemgV8faj6hqvYkeTbwmSS3V9XXVqiufwOuqKpHkryOwV89Lxpx3UnWNWsL8KGqemyobVLjNYrDsX+NLMkLGYT7rw41v6CN13HAdUm+0o5sV8KtwLOq6uEk5wD/CmxilYwXg1My/1lVw0f5Ex2vJE9j8Gbyxqr67tzF86wytv1rNR+5j3ILgx/2SbIG+CkGf55N8vYHI712khcDbwV+q6oemW2vqj3t+R7gswze0Vekrqr69lAt7wV+adR1J1nXkC3M+ZN5guM1ioPVfthvr5HkF4H3AZur6tuz7UPjtQ/4KOM7HbmgqvpuVT3cpq8FjkyyllUwXs2h9q+xj1eSIxkE+wer6iPzdJns/jXuCwljvCCxhsGFhBN5/CLMyXP6XMATL6he3aZP5okXVO9hfBdUR6nrVAYXkDbNaT8GOKpNrwXuZkwXlkasa93Q9MuBG+vxCzhfb/Ud06aPXam6Wr+fZ3BxKysxXkPb2MjBLxC+jCde8Lp50uM1Yl0nMLiO9Pw57U8Fnj40/QXg7BWs66dnf38MQvIbbexG2gcmVVdbPnvg99SVGK/2c38A+NtD9Jno/jW2wZ3Eg8HV5K8yCMq3tra/ZHA0DPBk4F/ajn4z8Oyhdd/a1rsLeOkK1/Vp4H7gtva4prU/H7i97dy3A+evcF1/DdzZtn8D8AtD6/5BG8ddwGtWsq42/3bgojnrTXq8rgD2Av/H4GjpfOB1wOva8jD4T2e+1rY/vULjtVBd7wMeHNq/Zlr7s9tYfan9nt+6wnW9fmj/upGhN5/59oGVqqv1eTWDD1kMrzex8WJwqqyAHUO/p3NWcv/y9gOS1KHVfM5dkrREhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nq0P8DPQZCvB9wts0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "lg_err = preds_df3[np.sqrt(preds_df3['cont_error']**2)>.15]\n",
    "sm_err = preds_df3[np.sqrt(preds_df3['cont_error']**2)<=.1]\n",
    "\n",
    "plt.hist(lg_err['t3'], bins=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 387,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAD5CAYAAAA+0W6bAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAOAElEQVR4nO3dX4yc1XnH8e+vNpRAgoCyWC7GXSJZFBQJiEaIFilqcVLREsW+CBFRi1apJd+kKWkjJQ43uemFkar8uaiiWoZkpVL+yAHZIlEa5IBopQplDagBnAjqWuDi4E2AhuSi1OnTi33dWOuxd7y7M+Oz+/1IaOY9+747z6s1Px8/e85MqgpJUnt+Y9wFSJIWxwCXpEYZ4JLUKANckhplgEtSowxwSWrU2kFOSnIJsBv4AFDAnwM/Bh4GJoHDwCeq6q0zfZ/LL7+8JicnF1+tJK1CBw4c+GlVTcwfzyDrwJNMA/9cVbuTnA9cCNwDvFlVO5PsAC6tqi+c6fv0er2amZlZ3B1I0iqV5EBV9eaPL9hCSXIx8CHgPoCqereq3ga2ANPdadPA1uUrV5K0kEF64O8HZoFvJHkuye4kFwHrquooQPd4xRDrlCTNM0iArwU+CHy9qm4EfgnsGPQFkmxPMpNkZnZ2dpFlSpLmGyTAjwBHquqZ7ngPc4H+RpL1AN3jsX4XV9WuqupVVW9i4pQevCRpkRYM8Kr6CfBakmu6oc3AS8A+YKobmwL2DqVCSVJfAy0jBD4DPNCtQDkEfIq58H8kyTbgVeCO4ZQoSepnoACvqueBU5awMDcblySNgTsxJalRBrgkNWrQHviKN7nj233HD++8fVmvkaTl4gxckhplgEtSowxwSWqUAS5JjTLAJalRBrgkNcplhAs43VJBSRo3Z+CS1CgDXJIaZYBLUqMMcElqlAEuSY0ywCWpUS4jbJDvgigJnIFLUrMMcElqlAEuSY1adT1wt8ZLWimcgUtSowxwSWqUAS5JjTLAJalRBrgkNcoAl6RGDbSMMMlh4B3gV8DxquoluQx4GJgEDgOfqKq3hlOmJGm+s5mB/2FV3VBVve54B7C/qjYB+7tjSdKILKWFsgWY7p5PA1uXXo4kaVCD7sQs4HtJCvj7qtoFrKuqowBVdTTJFf0uTLId2A6wcePGZSj53Oe7BUoahUED/Jaqer0L6SeS/GjQF+jCfhdAr9erRdQoSepjoBZKVb3ePR4DHgNuAt5Ish6gezw2rCIlSadaMMCTXJTkfSeeA38EvADsA6a606aAvcMqUpJ0qkFaKOuAx5KcOP8fq+q7SX4APJJkG/AqcMfwypQkzbdggFfVIeD6PuM/AzYPoyhJ0sLciSlJjTLAJalRBrgkNcoAl6RGGeCS1KhV96HG5yK33ktaDGfgktQoA1ySGmWAS1KjDHBJapQBLkmNMsAlqVEGuCQ1ygCXpEYZ4JLUKANckhplgEtSowxwSWqUAS5JjTLAJalRBrgkNcoAl6RGGeCS1CgDXJIaZYBLUqMMcElqlAEuSY0aOMCTrEnyXJLHu+OrkzyT5OUkDyc5f3hlSpLmO5sZ+N3AwZOO7wW+UlWbgLeAbctZmCTpzAYK8CQbgNuB3d1xgFuBPd0p08DWYRQoSepv7YDnfRX4PPC+7vi3gLer6nh3fAS4st+FSbYD2wE2bty4+EpXgMkd3x53CZJWkAVn4Ek+ChyrqgMnD/c5tfpdX1W7qqpXVb2JiYlFlilJmm+QGfgtwMeS/AlwAXAxczPyS5Ks7WbhG4DXh1emJGm+BWfgVfXFqtpQVZPAncD3q+pPgSeBj3enTQF7h1alJOkUS1kH/gXgr5O8wlxP/L7lKUmSNIhBf4kJQFU9BTzVPT8E3LT8JUmSBuFOTElqlAEuSY0ywCWpUQa4JDXKAJekRhngktQoA1ySGmWAS1Kjzmojj0bLdy+UdCbOwCWpUQa4JDXKAJekRhngktQoA1ySGmWAS1Kjml9GeLqldod33j7iSiRptJyBS1KjDHBJapQBLkmNMsAlqVEGuCQ1ygCXpEYZ4JLUqObXgevXXBMvrS7OwCWpUQa4JDVqwRZKkguAp4Hf7M7fU1VfSnI18BBwGfAscFdVvTvMYs+Gn2bza7ZWpJVpkBn4fwO3VtX1wA3AbUluBu4FvlJVm4C3gG3DK1OSNN+CAV5zftEdntf9V8CtwJ5ufBrYOpQKJUl9DdQDT7ImyfPAMeAJ4N+Bt6vqeHfKEeDK4ZQoSepnoGWEVfUr4IYklwCPAdf2O63ftUm2A9sBNm7cuMgyNUr2zKU2nNUqlKp6G3gKuBm4JMmJvwA2AK+f5ppdVdWrqt7ExMRSapUknWTBAE8y0c28SfIe4MPAQeBJ4OPdaVPA3mEVKUk61SAtlPXAdJI1zAX+I1X1eJKXgIeS/A3wHHDfEOuUJM2zYIBX1b8BN/YZPwTcNIyiJEkLcyemJDXKAJekRhngktQoA1ySGmWAS1KjDHBJapQBLkmNMsAlqVEGuCQ1ygCXpEYZ4JLUKANckhplgEtSowxwSWqUAS5JjTLAJalRBrgkNcoAl6RGGeCS1CgDXJIaZYBLUqMMcElqlAEuSY0ywCWpUQa4JDXKAJekRhngktSoBQM8yVVJnkxyMMmLSe7uxi9L8kSSl7vHS4dfriTphEFm4MeBz1XVtcDNwKeTXAfsAPZX1SZgf3csSRqRBQO8qo5W1bPd83eAg8CVwBZgujttGtg6rCIlSac6qx54kkngRuAZYF1VHYW5kAeuOM0125PMJJmZnZ1dWrWSpP83cIAneS/wLeCzVfXzQa+rql1V1auq3sTExGJqlCT1MVCAJzmPufB+oKoe7YbfSLK++/p64NhwSpQk9TPIKpQA9wEHq+rLJ31pHzDVPZ8C9i5/eZKk01k7wDm3AHcBP0zyfDd2D7ATeCTJNuBV4I7hlKhhmdzx7XGXIGkJFgzwqvoXIKf58ublLUeSNCh3YkpSowxwSWrUID1wCVhcz/zwztuHUIkkcAYuSc0ywCWpUQa4JDXKAJekRhngktQoA1ySGuUyQg3V6ZYeLtfywmF/f+lc5gxckhplgEtSowxwSWqUAS5JjTLAJalRBrgkNcplhBoLl/9JS+cMXJIaZYBLUqNsoeicYmtFGpwzcElqlAEuSY0ywCWpUc30wBfzgbpaOfz5S6dyBi5JjTLAJalRBrgkNWrBAE9yf5JjSV44aeyyJE8kebl7vHS4ZUqS5htkBv5N4LZ5YzuA/VW1CdjfHUuSRmjBAK+qp4E35w1vAaa759PA1mWuS5K0gMX2wNdV1VGA7vGK052YZHuSmSQzs7Ozi3w5SdJ8Q/8lZlXtqqpeVfUmJiaG/XKStGosNsDfSLIeoHs8tnwlSZIGsdgA3wdMdc+ngL3LU44kaVCDLCN8EPhX4JokR5JsA3YCH0nyMvCR7liSNEILvhdKVX3yNF/avMy1SJLOgjsxJalRzbwbobQc/MQfrSTOwCWpUQa4JDXKAJekRtkDl7A3rjY5A5ekRhngktQoWyhakfwQZK0GzsAlqVEGuCQ1ygCXpEbZA5dGZFxLFV0iuXI5A5ekRhngktQoA1ySGmUPXDqDs+0fu/5co+QMXJIaZYBLUqNsoUiLMIpWyWpc/rca73kpnIFLUqMMcElqlAEuSY2yBy41ZlxLFe1Pn3ucgUtSowxwSWpUqmrxFye3AV8D1gC7q2rnmc7v9Xo1MzOzqNdyh5uk+c619s2w2kxJDlRVb/74omfgSdYAfwf8MXAd8Mkk1y2+REnS2VhKC+Um4JWqOlRV7wIPAVuWpyxJ0kKWEuBXAq+ddHykG5MkjcBSlhGmz9gpDfUk24Ht3eEvkvwM+OkSXrc1l+P9rmTe7xjl3qF++2W712Wo83f6DS4lwI8AV510vAF4ff5JVbUL2HXiOMlMv2b8SuX9rmze78rVwr0upYXyA2BTkquTnA/cCexbnrIkSQtZ9Ay8qo4n+Qvgn5hbRnh/Vb24bJVJks5oSVvpq+o7wHfO8rJdC5+yoni/K5v3u3Kd8/e6pI08kqTxcSu9JDVqpAGe5LYkP07ySpIdo3ztUUtyVZInkxxM8mKSu8dd07AlWZPkuSSPj7uWYUtySZI9SX7U/Yx/b9w1DVOSv+r+HL+Q5MEkF4y7puWU5P4kx5K8cNLYZUmeSPJy93jpOGvsZ2QBvgq33h8HPldV1wI3A59e4fcLcDdwcNxFjMjXgO9W1e8C17OC7zvJlcBfAr2q+gBzixbuHG9Vy+6bwG3zxnYA+6tqE7C/Oz6njHIGvqq23lfV0ap6tnv+DnP/g6/YnapJNgC3A7vHXcuwJbkY+BBwH0BVvVtVb4+3qqFbC7wnyVrgQvrs+WhZVT0NvDlveAsw3T2fBraOtKgBjDLAV+3W+ySTwI3AM+OtZKi+Cnwe+N9xFzIC7wdmgW90LaPdSS4ad1HDUlX/Cfwt8CpwFPivqvreeKsaiXVVdRTmJmTAFWOu5xSjDPCBtt6vNEneC3wL+GxV/Xzc9QxDko8Cx6rqwLhrGZG1wAeBr1fVjcAvOQf/eb1cut7vFuBq4LeBi5L82XirEow2wAfaer+SJDmPufB+oKoeHXc9Q3QL8LEkh5lrjd2a5B/GW9JQHQGOVNWJf1HtYS7QV6oPA/9RVbNV9T/Ao8Dvj7mmUXgjyXqA7vHYmOs5xSgDfFVtvU8S5nqkB6vqy+OuZ5iq6otVtaGqJpn7uX6/qlbsDK2qfgK8luSabmgz8NIYSxq2V4Gbk1zY/bnezAr+pe1J9gFT3fMpYO8Ya+lrZB9qvAq33t8C3AX8MMnz3dg93e5Vte8zwAPdZOQQ8Kkx1zM0VfVMkj3As8ytrnqOBnYpno0kDwJ/AFye5AjwJWAn8EiSbcz9JXbH+Crsz52YktQod2JKUqMMcElqlAEuSY0ywCWpUQa4JDXKAJekRhngktQoA1ySGvV/jAaBLzCA00AAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "_=plt.hist(lg_err['gap1'], bins=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 388,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAOQ0lEQVR4nO3df4xlZX3H8fenrIhgDSiDwV3oYrJBqanBTChKYohrUxAD/AEJprUbSrNpgoo/GlnsH/yLqfFHk9ZkA+g2JfzoSgMRYyUUYvqHW3eFyI+FskG6jKzuGAWNJtVtv/1jzsbZ3TvO3Hvu3Tvz7PuVbO49zzn3nm9Odj7zzHPOeU6qCklSW35v2gVIksbPcJekBhnuktQgw12SGmS4S1KD1k27AIAzzzyzNm7cOO0yJGlN2bNnz0+qambQulUR7hs3bmT37t3TLkOS1pQk/73UOodlJKlBhrskNchwl6QGGe6S1CDDXZIaZLhLUoOWDfckdyY5mOSpRW1/l+TZJN9P8q9JTl+07pYk+5I8l+RPJ1W4JGlpK+m5fxW47Ki2h4F3VNUfAf8F3AKQ5ALgOuAPu8/8Y5KTxlatJGlFlg33qvo28NOj2r5VVYe6xe8AG7r3VwH3VNX/VNUPgH3ARWOsV5K0AuO4Q/UvgXu79+tZCPvD5rq2YyTZCmwFOPfcc8dQxspt3PbQwPYXb7viuNYhSZPS64Rqkr8FDgF3HW4asNnARz1V1faqmq2q2ZmZgVMjSJJGNHLPPckW4IPA5vrts/rmgHMWbbYBeHn08iRJoxip557kMuBm4Mqq+tWiVQ8C1yV5bZLzgE3Af/YvU5I0jGV77knuBi4FzkwyB9zKwtUxrwUeTgLwnar666p6Osl9wDMsDNfcWFX/O6niJUmD5bcjKtMzOztbx3PK36VOqC7FE62SVqMke6pqdtA671CVpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lq0Dges9c8H8snaa2x5y5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDVo2XBPcmeSg0meWtT2xiQPJ3m+ez2ja0+Sv0+yL8n3k7xrksVLkgZbSc/9q8BlR7VtAx6pqk3AI90ywOXApu7fVuDL4ylTkjSMZcO9qr4N/PSo5quAHd37HcDVi9r/qRZ8Bzg9ydnjKlaStDKjjrm/uaoOAHSvZ3Xt64GXFm0317UdI8nWJLuT7J6fnx+xDEnSIOM+oZoBbTVow6raXlWzVTU7MzMz5jIk6cQ2arj/+PBwS/d6sGufA85ZtN0G4OXRy5MkjWLUcH8Q2NK93wI8sKj9L7qrZi4GXj08fCNJOn6WfcxekruBS4Ezk8wBtwK3AfcluQHYD1zbbf4N4APAPuBXwPUTqFmStIxlw72qPrTEqs0Dti3gxr5FSZL68Q5VSWrQsj13LW3jtocGtr942xXHuRJJOpI9d0lqkOEuSQ0y3CWpQY65T4Bj8ZKmzZ67JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUoKbnlllqjhdJap09d0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDeoV7kk8keTrJU0nuTnJKkvOS7EryfJJ7k5w8rmIlSSszcrgnWQ98DJitqncAJwHXAZ8FvlBVm4CfATeMo1BJ0sr1HZZZB7wuyTrgVOAA8D5gZ7d+B3B1z31IkoY0crhX1Q+BzwH7WQj1V4E9wCtVdajbbA5YP+jzSbYm2Z1k9/z8/KhlSJIG6DMscwZwFXAe8BbgNODyAZvWoM9X1faqmq2q2ZmZmVHLkCQN0GdY5v3AD6pqvqp+A9wPvAc4vRumAdgAvNyzRknSkPqE+37g4iSnJgmwGXgGeBS4pttmC/BAvxIlScPqM+a+i4UTp98Dnuy+aztwM/DJJPuANwF3jKFOSdIQej2so6puBW49qvkF4KI+3ytJ6sc7VCWpQU0/Zm+1Gfaxfy/edsWEKpHUOnvuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBTvm7ii01RbBTAUtajj13SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoN6hXuS05PsTPJskr1J3p3kjUkeTvJ893rGuIqVJK1M3577l4BvVtXbgHcCe4FtwCNVtQl4pFuWJB1HI4d7kjcA7wXuAKiqX1fVK8BVwI5usx3A1X2LlCQNp0/P/a3APPCVJI8nuT3JacCbq+oAQPd61qAPJ9maZHeS3fPz8z3KkCQdrU+4rwPeBXy5qi4EfskQQzBVtb2qZqtqdmZmpkcZkqSj9Qn3OWCuqnZ1yztZCPsfJzkboHs92K9ESdKwRg73qvoR8FKS87umzcAzwIPAlq5tC/BArwolSUPrO5/7R4G7kpwMvABcz8IvjPuS3ADsB67tuQ9J0pB6hXtVPQHMDli1uc/3DmOpB1pI0onMO1QlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNWjftAjRdG7c9NLD9xduuOM6VSBone+6S1KDe4Z7kpCSPJ/l6t3xekl1Jnk9yb5KT+5cpSRrGOHruNwF7Fy1/FvhCVW0CfgbcMIZ9SJKG0Cvck2wArgBu75YDvA/Y2W2yA7i6zz4kScPre0L1i8Cngd/vlt8EvFJVh7rlOWD9oA8m2QpsBTj33HN7lnFiGfYk6FLbS2rXyD33JB8EDlbVnsXNAzatQZ+vqu1VNVtVszMzM6OWIUkaoE/P/RLgyiQfAE4B3sBCT/70JOu63vsG4OX+ZUqShjFyuFfVLcAtAEkuBf6mqv4syb8A1wD3AFuAB8ZQp1bA4RdJh03iOvebgU8m2cfCGPwdE9iHJOl3GMsdqlX1GPBY9/4F4KJxfK8kaTTeoSpJDTLcJalBThymsXACMml1secuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGuSlkBrISxultc2euyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQl0JqorykUpoOe+6S1CDDXZIaZLhLUoMcc9dQlhpDl7S62HOXpAYZ7pLUIIdltCZ4SaU0HHvuktSgkcM9yTlJHk2yN8nTSW7q2t+Y5OEkz3evZ4yvXEnSSvTpuR8CPlVVbwcuBm5McgGwDXikqjYBj3TLkqTjaOQx96o6ABzo3v8iyV5gPXAVcGm32Q7gMeDmXlWqOY6hS5M1ljH3JBuBC4FdwJu74D/8C+CsJT6zNcnuJLvn5+fHUYYkqdM73JO8Hvga8PGq+vlKP1dV26tqtqpmZ2Zm+pYhSVqkV7gneQ0LwX5XVd3fNf84ydnd+rOBg/1KlCQNq8/VMgHuAPZW1ecXrXoQ2NK93wI8MHp5kqRR9LmJ6RLgw8CTSZ7o2j4D3Abcl+QGYD9wbb8SJUnD6nO1zH8AWWL15lG/V5LUn3eoSlKDDHdJapDhLkkNclZInVB+18NGvDtWLbHnLkkNsueuVWXYx/g5R400mOEudfxFoZY4LCNJDbLnLo2ZfwFoNbDnLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhrkpZBq0rB3ukqtsecuSQ2y5y4tw5uStBYZ7tIa4y8brYTDMpLUIHvu0nEy7Elee+Lqw567JDXInrs0oklfbjmu73eM/sRkz12SGmTPXdIRpvkQcf/KGB/DXWrEuJ4/ezz4rNzJm9iwTJLLkjyXZF+SbZPajyTpWBPpuSc5CfgH4E+AOeC7SR6sqmcmsT9JOt6G/WvieP/1Mame+0XAvqp6oap+DdwDXDWhfUmSjpKqGv+XJtcAl1XVX3XLHwb+uKo+smibrcDWbvF84LkRdnUm8JOe5bbE43Ekj8eRPB7HWuvH5A+qambQikmdUM2AtiN+i1TVdmB7r50ku6tqts93tMTjcSSPx5E8Hsdq+ZhMalhmDjhn0fIG4OUJ7UuSdJRJhft3gU1JzktyMnAd8OCE9iVJOspEhmWq6lCSjwD/BpwE3FlVT09gV72GdRrk8TiSx+NIHo9jNXtMJnJCVZI0Xc4tI0kNMtwlqUFrMtyd2uBISc5J8miSvUmeTnLTtGtaDZKclOTxJF+fdi3TluT0JDuTPNv9P3n3tGuapiSf6H5Wnkpyd5JTpl3TuK25cF80tcHlwAXAh5JcMN2qpu4Q8KmqejtwMXCjxwSAm4C90y5ilfgS8M2qehvwTk7g45JkPfAxYLaq3sHCRR/XTbeq8Vtz4Y5TGxyjqg5U1fe6979g4Qd3/XSrmq4kG4ArgNunXcu0JXkD8F7gDoCq+nVVvTLdqqZuHfC6JOuAU2nwPpy1GO7rgZcWLc9xggfZYkk2AhcCu6ZbydR9Efg08H/TLmQVeCswD3ylG6a6Pclp0y5qWqrqh8DngP3AAeDVqvrWdKsav7UY7stObXCiSvJ64GvAx6vq59OuZ1qSfBA4WFV7pl3LKrEOeBfw5aq6EPglcMKeq0pyBgt/7Z8HvAU4LcmfT7eq8VuL4e7UBgMkeQ0LwX5XVd0/7Xqm7BLgyiQvsjBs974k/zzdkqZqDpirqsN/ze1kIexPVO8HflBV81X1G+B+4D1Trmns1mK4O7XBUZKEhfHUvVX1+WnXM21VdUtVbaiqjSz8//j3qmquZ7ZSVfUj4KUk53dNm4ET+dkK+4GLk5za/exspsETzGvuMXvHcWqDteQS4MPAk0me6No+U1XfmGJNWl0+CtzVdYheAK6fcj1TU1W7kuwEvsfClWaP0+A0BE4/IEkNWovDMpKkZRjuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUH/Dx08cId/WXY+AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "_=plt.hist(sm_err['gap1'], bins=50)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Calculate MAE only for observations with:\n",
    " - actual conversion==1: we only care about attributing people who converted\n",
    " - t1 channel does not equal t2 channel: there is no uncertainty of attribution in cases where t1 channel==t2 channel\n",
    " - predicted probability >0.03: These are likely noisier since the model found no signal for conversion. This was chosen arbitrarily ahead of time and does not change for each run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 389,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAOAUlEQVR4nO3df6zddX3H8ecLpiZTFiG9kK6UXWe6ZMU42O4YicmCYZkIW4sZGEimxbDULRA1849Vt0TnQtL9UOMyR1YGsSYqsKmxG8wNGxbCH6gXwkDsiFU7qG3oVYyymLkV3vvjfhsP5d6ec+/51fu5z0dycs73cz7fc96fnOZ1P/2c7/d7UlVIktpyxrQLkCSNnuEuSQ0y3CWpQYa7JDXIcJekBv3UtAsA2LBhQ83Ozk67DElaUx5++OHvVtXMUs+dFuE+OzvL/Pz8tMuQpDUlyX8t95zLMpLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1KDT4gzV083srnsG6ndo91VjrkSSVseZuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CCvLTMEr0Ej6XTlzF2SGmS4S1KDDHdJapDhLkkN6hvuSTYnuT/JgSRPJHl31/7BJN9J8mh3u7Jnn/clOZjkySRvGucAJEkvNcjRMseB91bVI0nOAh5Ocl/33Eer6q96OyfZClwHXAj8LPClJL9QVc+PsnBJ0vL6ztyr6mhVPdI9fg44AGw6xS7bgTur6sdV9W3gIHDJKIqVJA1mRWvuSWaBi4Evd003J3ksyR1Jzu7aNgFP9+x2mFP/MZAkjdjA4Z7kVcBngfdU1Q+BW4HXAhcBR4EPn+i6xO61xOvtTDKfZH5hYWHFhUuSljdQuCd5GYvB/qmq+hxAVT1TVc9X1QvAbfxk6eUwsLln9/OBIye/ZlXtqaq5qpqbmZkZZgySpJMMcrRMgNuBA1X1kZ72jT3d3gJ8rXu8D7guySuSvAbYAnxldCVLkvoZ5GiZNwBvAx5P8mjX9n7g+iQXsbjkcgh4J0BVPZHkbuDrLB5pc5NHykjSZPUN96p6kKXX0e89xT63ALcMUZckaQieoSpJDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1aJAfyG7G7K57pl2CJE2EM3dJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWpQ33BPsjnJ/UkOJHkiybu79nOS3JfkG9392V17kvx1koNJHkvyy+MehCTpxQaZuR8H3ltVvwhcCtyUZCuwC9hfVVuA/d02wJuBLd1tJ3DryKuWJJ1S33CvqqNV9Uj3+DngALAJ2A7s7brtBa7uHm8HPlmLHgJenWTjyCuXJC1rRWvuSWaBi4EvA+dV1VFY/AMAnNt12wQ83bPb4a7t5NfamWQ+yfzCwsLKK5ckLWvgcE/yKuCzwHuq6oen6rpEW72koWpPVc1V1dzMzMygZUiSBjBQuCd5GYvB/qmq+lzX/MyJ5Zbu/ljXfhjY3LP7+cCR0ZQrSRrEIEfLBLgdOFBVH+l5ah+wo3u8A/hCT/vbu6NmLgV+cGL5RpI0GYP8WMcbgLcBjyd5tGt7P7AbuDvJjcBTwLXdc/cCVwIHgR8B7xhpxZKkvvqGe1U9yNLr6ACXL9G/gJuGrEuSNATPUJWkBhnuktQgw12SGmS4S1KDBjlaRkOa3XXPQP0O7b5qzJVIWi+cuUtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUH+QPZpxB/SljQqztwlqUGGuyQ1qG+4J7kjybEkX+tp+2CS7yR5tLtd2fPc+5IcTPJkkjeNq3BJ0vIGmbl/ArhiifaPVtVF3e1egCRbgeuAC7t9/jbJmaMqVpI0mL7hXlUPAM8O+HrbgTur6sdV9W3gIHDJEPVJklZhmDX3m5M81i3bnN21bQKe7ulzuGt7iSQ7k8wnmV9YWBiiDEnSyVYb7rcCrwUuAo4CH+7as0TfWuoFqmpPVc1V1dzMzMwqy5AkLWVV4V5Vz1TV81X1AnAbP1l6OQxs7ul6PnBkuBIlSSu1qnBPsrFn8y3AiSNp9gHXJXlFktcAW4CvDFeiJGml+p6hmuQzwGXAhiSHgQ8AlyW5iMUll0PAOwGq6okkdwNfB44DN1XV8+MpXZK0nL7hXlXXL9F8+yn63wLcMkxRkqTheIaqJDXIcJekBhnuktQgw12SGmS4S1KD/LGONcgf9ZDUjzN3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkGeoakU8O1ZaG5y5S1KDDHdJapDhLkkNMtwlqUF+odqwQb/8lNQeZ+6S1CDDXZIa5LKMpsrj5qXxcOYuSQ0y3CWpQYa7JDXIcJekBvX9QjXJHcBvAceq6nVd2znAXcAscAh4a1V9P0mAjwFXAj8CbqiqR8ZTuk5nHmMvTdcgM/dPAFec1LYL2F9VW4D93TbAm4Et3W0ncOtoypQkrUTfcK+qB4BnT2reDuztHu8Fru5p/2Qtegh4dZKNoypWkjSY1a65n1dVRwG6+3O79k3A0z39DndtL5FkZ5L5JPMLCwurLEOStJRRf6GaJdpqqY5Vtaeq5qpqbmZmZsRlSNL6ttpwf+bEckt3f6xrPwxs7ul3PnBk9eVJklZjteG+D9jRPd4BfKGn/e1ZdCnwgxPLN5KkyRnkUMjPAJcBG5IcBj4A7AbuTnIj8BRwbdf9XhYPgzzI4qGQ7xhDzZKkPvqGe1Vdv8xTly/Rt4Cbhi1KkjQcz1CVpAYZ7pLUIMNdkhq05n+sw2uYSNJLOXOXpAYZ7pLUoDW/LKP1wd9alVbGmbskNchwl6QGGe6S1CDDXZIa5BeqUh9+mau1yHDXuuTJb2qdyzKS1CBn7mqKM3JpkTN3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQV5bRhoRLw2s04kzd0lq0FAz9ySHgOeA54HjVTWX5BzgLmAWOAS8taq+P1yZkqSVGMXM/Y1VdVFVzXXbu4D9VbUF2N9tS5ImaBzLMtuBvd3jvcDVY3gPSdIpDBvuBfxbkoeT7OzazquqowDd/blDvockaYWGPVrmDVV1JMm5wH1J/nPQHbs/BjsBLrjggiHLkCT1Gircq+pId38syeeBS4BnkmysqqNJNgLHltl3D7AHYG5uroapQ2qRh1ZqGKsO9ySvBM6oque6x78JfAjYB+wAdnf3XxhFoVIr/J1XTcIwM/fzgM8nOfE6n66qLyb5KnB3khuBp4Brhy9T0nKc4Wspqw73qvoW8EtLtH8PuHyYoiRJw/EMVUlqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgf6xD0qp5jP3py5m7JDXImbukF/HyCG0w3KV1wtBeXwx3SWPn2vzkueYuSQ0y3CWpQYa7JDXIcJekBvmFqqRmrecvcg13SWuOh3X257KMJDXImbukda/F5RvDXdJpw+WW0XFZRpIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBnmcuySNwbRPjHLmLkkNGtvMPckVwMeAM4G/r6rd43ovSZqEtXQG7Vhm7knOBD4OvBnYClyfZOs43kuS9FLjWpa5BDhYVd+qqv8F7gS2j+m9JEknGdeyzCbg6Z7tw8Cv9XZIshPY2W3+d5Inx1TL6WQD8N1pFzEljn19cux95M+Heo+fW+6JcYV7lmirF21U7QH2jOn9T0tJ5qtqbtp1TINjd+zrzbTHPq5lmcPA5p7t84EjY3ovSdJJxhXuXwW2JHlNkpcD1wH7xvRekqSTjGVZpqqOJ7kZ+FcWD4W8o6qeGMd7rTHrahnqJI59fXLsU5Kq6t9LkrSmeIaqJDXIcJekBhnuY5DkiiRPJjmYZNcSz/96kkeSHE9yzTRqHJcBxv6HSb6e5LEk+5Mse5zuWjPA2H8/yeNJHk3yYEtnbfcbe0+/a5JUkmYOjxzgc78hyUL3uT+a5PcmUlhVeRvhjcUvkL8J/DzwcuA/gK0n9ZkFXg98Erhm2jVPeOxvBH66e/wHwF3TrnuCY/+ZnsfbgC9Ou+5Jjb3rdxbwAPAQMDftuif4ud8A/M2ka3PmPnp9L71QVYeq6jHghWkUOEaDjP3+qvpRt/kQi+dAtGCQsf+wZ/OVnHRi3xo26OVG/gz4C+B/JlncmJ22l1ox3EdvqUsvbJpSLZO20rHfCPzLWCuanIHGnuSmJN9kMeTeNaHaxq3v2JNcDGyuqn+eZGETMOi/+d/pliL/McnmJZ4fOcN99PpeeqFhA489ye8Cc8BfjrWiyRlo7FX18ap6LfBHwJ+MvarJOOXYk5wBfBR478QqmpxBPvd/Amar6vXAl4C9Y68Kw30c1vOlFwYae5LfAP4Y2FZVP55QbeO20s/9TuDqsVY0Of3GfhbwOuDfkxwCLgX2NfKlat/Pvaq+1/Pv/DbgVyZRmOE+euv50gt9x9799/zvWAz2Y1OocVwGGfuWns2rgG9MsL5xOuXYq+oHVbWhqmarapbF71q2VdX8dModqUE+9409m9uAA5MozN9QHbFa5tILST4EzFfVviS/CnweOBv47SR/WlUXTrHskRhk7Cwuw7wK+IckAE9V1bapFT0iA4795u5/Lf8HfB/YMb2KR2fAsTdpwLG/K8k24DjwLItHz4ydlx+QpAa5LCNJDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoP+H5tzuPVrk1veAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "_=plt.hist(preds_df3['pred_prob'], bins=30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 390,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pct obs over min_p: 1.0\n"
     ]
    }
   ],
   "source": [
    "min_p = .03\n",
    "print('pct obs over min_p:', round(len(preds_df3[preds_df3['pred_prob']>min_p])/len(preds_df3), 3 ) )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Predicted prob and the DLAW error is strongly negatively correlated indicating more certainty with higher predictions of conversion."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 391,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Corr between predicted prob and magnitude of DLAW error: -0.4521\n",
      "Corr between predicted magnitude of t1_attn and magnitude of DLAW error: -0.125\n"
     ]
    }
   ],
   "source": [
    "print('Corr between predicted prob and magnitude of DLAW error:', round(np.corrcoef(preds_df3['pred_prob'], preds_df3['cont_error'])[0][1], 4))\n",
    "print('Corr between predicted magnitude of t1_attn and magnitude of DLAW error:', round(np.corrcoef(preds_df3['t1_attn'], preds_df3['cont_error'])[0][1], 4))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 392,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DLAW Stats:\n",
      " Mean AE: 0.11 , SD AE: 0.083 , Median AE 0.097 95% CI Error: 0.246\n",
      "\n",
      "50/50 Split Stats:\n",
      " Mean AE: 0.248 , Median AE 0.204 95% CI Error: 0.438\n",
      "\n",
      "Last Touched Stats:\n",
      " Mean AE: 0.345 , Median AE 0.364 95% CI Error: 0.601\n",
      "\n",
      "First Touched Stats:\n",
      " Mean AE: 0.648 , Median AE 0.762 95% CI Error: 1.034\n"
     ]
    }
   ],
   "source": [
    "dlaw_mae = preds_df3[(preds_df3['actual']==1)  & (preds_df3['pred_prob']>min_p)]['cont_error'].apply(lambda x: abs(x)).mean()\n",
    "dlaw_medae = preds_df3[(preds_df3['actual']==1) & (preds_df3['pred_prob']>min_p)]['cont_error'].apply(lambda x: abs(x)).median()\n",
    "dlaw_err_sd = preds_df3[(preds_df3['actual']==1)  & (preds_df3['pred_prob']>min_p)]['cont_error'].apply(lambda x: abs(x)).std()\n",
    "dlaw_95CI = dlaw_mae+(1.64*dlaw_err_sd)\n",
    "\n",
    "even_split_mae = preds_df3[(preds_df3['actual']==1)  & (preds_df3['pred_prob']>min_p)]['5050_cont_error'].apply(lambda x: abs(x)).mean()\n",
    "even_split_medae = preds_df3[(preds_df3['actual']==1)  & (preds_df3['pred_prob']>min_p)]['5050_cont_error'].apply(lambda x: abs(x)).median()\n",
    "even_split_err_sd = preds_df3[(preds_df3['actual']==1) & (preds_df3['pred_prob']>min_p)]['5050_cont_error'].apply(lambda x: abs(x)).std()\n",
    "even_split_95CI = even_split_mae+(1.64*even_split_err_sd)\n",
    "\n",
    "\n",
    "lt_mae = preds_df3[(preds_df3['actual']==1)  & (preds_df3['pred_prob']>min_p)]['last_touch_cont_error'].apply(lambda x: abs(x)).mean()\n",
    "lt_medae = preds_df3[(preds_df3['actual']==1) & (preds_df3['pred_prob']>min_p)]['last_touch_cont_error'].apply(lambda x: abs(x)).median()\n",
    "lt_err_sd = preds_df3[(preds_df3['actual']==1) & (preds_df3['pred_prob']>min_p)]['last_touch_cont_error'].apply(lambda x: abs(x)).std()\n",
    "lt_95CI = lt_mae+(1.64*lt_err_sd)\n",
    "\n",
    "\n",
    "ft_mae = preds_df3[(preds_df3['actual']==1)  & (preds_df3['pred_prob']>min_p)]['first_touch_cont_error'].apply(lambda x: abs(x)).mean()\n",
    "ft_medae = preds_df3[(preds_df3['actual']==1) & (preds_df3['pred_prob']>min_p)]['first_touch_cont_error'].apply(lambda x: abs(x)).median()\n",
    "ft_err_sd = preds_df3[(preds_df3['actual']==1) & (preds_df3['pred_prob']>min_p)]['first_touch_cont_error'].apply(lambda x: abs(x)).std()\n",
    "ft_95CI = ft_mae+(1.64*ft_err_sd)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "print('DLAW Stats:\\n', 'Mean AE:', round(dlaw_mae, 3),', SD AE:',round(dlaw_err_sd, 3), ', Median AE', round(dlaw_medae, 3), '95% CI Error:', round(dlaw_95CI, 3))\n",
    "print('\\n50/50 Split Stats:\\n', 'Mean AE:', round(even_split_mae, 3), ', Median AE', round(even_split_medae, 3), '95% CI Error:', round(even_split_95CI, 3))\n",
    "print('\\nLast Touched Stats:\\n', 'Mean AE:', round(lt_mae, 3), ', Median AE', round(lt_medae, 3), '95% CI Error:', round(lt_95CI, 3))\n",
    "print('\\nFirst Touched Stats:\\n', 'Mean AE:', round(ft_mae, 3), ', Median AE', round(ft_medae, 3), '95% CI Error:', round(ft_95CI, 3))\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['index', 't1', 't2', 't3', 'gender', 'gap1', 'log_gap1', 'gender_t2',\n",
       "       'gap2', 'log_gap2', 'gender_t3', 'gap3', 'log_gap3', 'epsilon',\n",
       "       'pred_prob', 'actual', 'pred', 't1_attn_unscaled', 't2_attn_unscaled',\n",
       "       't3_attn_unscaled', 'cont_fb', 'cont_gg', 'cont_sp', 't1_attn',\n",
       "       't2_attn', 't3_attn', 't1_cont', 't2_cont', 't3_cont', 't23attn',\n",
       "       'cont_error', '5050_cont_error', 'last_touch_cont_error',\n",
       "       'first_touch_cont_error'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 155,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds_df3.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### DLAW Distribution of Residuals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAP/ElEQVR4nO3df4xlZX3H8fdHqDZarD92NATYDpDVBo1d65SaGA0W2yI0oC1aNq2ipV1pJW2jf7hqU42NKf6gpI2KXSIBE0VUSqWCVqQqMSnqoCsuIrrQVVc27AoGbbE0i9/+MWf1OtzduTPn3vnxzPuV3Mw5zz3nnu+TO3z24Tk/JlWFJKktj1jpAiRJ42e4S1KDDHdJapDhLkkNMtwlqUFHrnQBABs2bKjp6emVLkOS1pRbbrnl+1U1Ney9VRHu09PTzM7OrnQZkrSmJPn2od5zWkaSGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhq0Ku5Q7Wt623VD23dfeMYyVyJJq4Mjd0lqkOEuSQ0y3CWpQYa7JDVowXBPclmSfUl2DrRdlWRH99qdZEfXPp3kxwPvvXeSxUuShhvlapnLgXcB7z/YUFV/eHA5yUXA/QPb31lVm8dVoCRp8RYM96q6Kcn0sPeSBHgp8FvjLUuS1EffOffnAvdU1bcG2o5P8pUkn0vy3EPtmGRrktkks/v37+9ZhiRpUN9w3wJcObC+F9hYVc8EXgN8MMljh+1YVduraqaqZqamhv4JQEnSEi053JMcCfw+cNXBtqp6sKru7ZZvAe4EntK3SEnS4vQZub8A+EZV7TnYkGQqyRHd8gnAJuCufiVKkhZrlEshrwT+E3hqkj1JzuveOoefn5IBeB5wa5KvAh8Fzq+q+8ZZsCRpYaNcLbPlEO2vGNJ2NXB1/7IkSX14h6okNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQaM8z33Nmt523dD23ReescyVSNLycuQuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGj/IHsy5LsS7JzoO3NSb6XZEf3On3gvdcn2ZXkjiS/O6nCJUmHNsrI/XLgtCHtF1fV5u51PUCSk4BzgKd1+7wnyRHjKlaSNJoFw72qbgLuG/HzzgI+VFUPVtV/AbuAk3vUJ0lagj5z7hckubWbtnl813YM8N2BbfZ0bQ+TZGuS2SSz+/fv71GGJGm+pYb7JcCJwGZgL3BR154h29awD6iq7VU1U1UzU1NTSyxDkjTMksK9qu6pqoeq6ifApfxs6mUPcNzApscCd/crUZK0WEsK9yRHD6y+GDh4Jc21wDlJHpXkeGAT8MV+JUqSFmvBv8SU5ErgFGBDkj3Am4BTkmxmbsplN/AqgKq6LcmHga8DB4BXV9VDkyldknQoC4Z7VW0Z0vy+w2z/VuCtfYqSJPXjHaqS1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgxYM9ySXJdmXZOdA2zuSfCPJrUmuSfK4rn06yY+T7Ohe751k8ZKk4UYZuV8OnDav7Qbg6VX1DOCbwOsH3ruzqjZ3r/PHU6YkaTEWDPequgm4b17bp6rqQLd6M3DsBGqTJC3ROObc/wT4xMD68Um+kuRzSZ47hs+XJC3SkX12TvJG4ADwga5pL7Cxqu5N8izgX5M8rap+OGTfrcBWgI0bN/YpQ5I0z5JH7knOBX4P+KOqKoCqerCq7u2WbwHuBJ4ybP+q2l5VM1U1MzU1tdQyJElDLCnck5wGvA44s6oeGGifSnJEt3wCsAm4axyFSpJGt+C0TJIrgVOADUn2AG9i7uqYRwE3JAG4ubsy5nnAW5IcAB4Czq+q+4Z+sCRpYhYM96raMqT5fYfY9mrg6r5FSZL68Q5VSWqQ4S5JDTLcJalBhrskNchwl6QG9bpDda2a3nbd0PbdF56xzJVI0mQ4cpekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDRop3JNclmRfkp0DbU9IckOSb3U/H9+1J8k/JdmV5NYkvz6p4iVJw406cr8cOG1e2zbgxqraBNzYrQO8ENjUvbYCl/QvU5K0GCOFe1XdBNw3r/ks4Ipu+QrgRQPt7685NwOPS3L0OIqVJI2mz5z7k6tqL0D380ld+zHAdwe229O1/ZwkW5PMJpndv39/jzIkSfNN4oRqhrTVwxqqtlfVTFXNTE1NTaAMSVq/+oT7PQenW7qf+7r2PcBxA9sdC9zd4ziSpEXqE+7XAud2y+cCHxtof3l31cyzgfsPTt9IkpbHkaNslORK4BRgQ5I9wJuAC4EPJzkP+A7wkm7z64HTgV3AA8Arx1yzJGkBI4V7VW05xFunDtm2gFf3KUqS1M9I4S4dNL3tuqHtuy88Y5krkXQ4hvsqtpJBeqhjS1obfLaMJDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkN8jr3dcKbj6T1xZG7JDXIcJekBhnuktQg59w1Fs7pS6uLI3dJapDhLkkNMtwlqUGGuyQ1yBOq65x/lENq05LDPclTgasGmk4A/hZ4HPBnwP6u/Q1Vdf2SK5QkLdqSw72q7gA2AyQ5AvgecA3wSuDiqnrnWCqUJC3auObcTwXurKpvj+nzJEk9jGvO/RzgyoH1C5K8HJgFXltVP5i/Q5KtwFaAjRs3jqkMOYcuCcYwck/ySOBM4CNd0yXAicxN2ewFLhq2X1Vtr6qZqpqZmprqW4YkacA4Ru4vBL5cVfcAHPwJkORS4ONjOIYGODqXtJBxhPsWBqZkkhxdVXu71RcDO8dwjGWx2ND0uSmSVqte4Z7k0cBvA68aaH57ks1AAbvnvSdJWga9wr2qHgCeOK/tZb0q0rrm0yWl8fDxA5LUIMNdkhpkuEtSgwx3SWqQ4S5JDfKRv6uANyVJGjdH7pLUIEfuWtO8Ll4azpG7JDXIcJekBhnuktQgw12SGmS4S1KDvFpGK2Kx1/Z7L4C0OIb7BHh5nqSVZrhrohxxSyvDOXdJapDhLkkNclqmB6ccJK1WhruadLh/eD2xrfWgd7gn2Q38CHgIOFBVM0meAFwFTAO7gZdW1Q/6HkuSNJpxjdyfX1XfH1jfBtxYVRcm2datv25Mx5J68VJVrQeTOqF6FnBFt3wF8KIJHUeSNMQ4wr2ATyW5JcnWru3JVbUXoPv5pPk7JdmaZDbJ7P79+8dQhiTpoHFMyzynqu5O8iTghiTfGGWnqtoObAeYmZmpMdSx6nl1jaTl0nvkXlV3dz/3AdcAJwP3JDkaoPu5r+9xJEmj6xXuSR6T5KiDy8DvADuBa4Fzu83OBT7W5ziSpMXpOy3zZOCaJAc/64NV9ckkXwI+nOQ84DvAS3oeR5K0CL3CvaruAn5tSPu9wKl9PluStHTeoSp1vP5dLfHBYZLUIMNdkhpkuEtSgwx3SWqQJ1SlBXiiVWuR4S6Nmf8YaDVwWkaSGmS4S1KDDHdJapBz7tIS+QhnrWaGu7RMPNGq5eS0jCQ1yHCXpAY5LSOtMKdrNAmO3CWpQYa7JDXIcJekBhnuktSgJYd7kuOSfCbJ7UluS/JXXfubk3wvyY7udfr4ypUkjaLP1TIHgNdW1ZeTHAXckuSG7r2Lq+qd/cuTJC3FksO9qvYCe7vlHyW5HThmXIVJkpZuLNe5J5kGngl8AXgOcEGSlwOzzI3ufzBkn63AVoCNGzeOowxJQxzuGTheS9+u3idUk/wScDXw11X1Q+AS4ERgM3Mj+4uG7VdV26tqpqpmpqam+pYhSRrQK9yT/AJzwf6BqvoXgKq6p6oeqqqfAJcCJ/cvU5K0GEuelkkS4H3A7VX1DwPtR3fz8QAvBnb2K1Fanxb7SGGnWDSoz5z7c4CXAV9LsqNrewOwJclmoIDdwKt6VShJWrQ+V8t8HsiQt65fejmSVgMfZrb2eYeqJDXIcJekBhnuktQg/1iH1Aj/YLcGOXKXpAYZ7pLUIKdlJI3MG6vWDkfuktQgw12SGuS0jLSOeYVNuxy5S1KDHLlLmpjFPqPGZ9qMjyN3SWqQ4S5JDTLcJalBzrlLWrOcoz80w13SslvsJZhesrl4TstIUoMcuUtqjs/AceQuSU2aWLgnOS3JHUl2Jdk2qeNIkh5uIuGe5Ajg3cALgZOALUlOmsSxJEkPN6k595OBXVV1F0CSDwFnAV+f0PEkacmWcjXOYh+hsNjP6WtS4X4M8N2B9T3Abw5ukGQrsLVb/e8kdyzxWBuA7y9x37VkPfTTPraj+X7mbePpY97Wa/dfOdQbkwr3DGmrn1up2g5s732gZLaqZvp+zmq3HvppH9uxHvq52vs4qROqe4DjBtaPBe6e0LEkSfNMKty/BGxKcnySRwLnANdO6FiSpHkmMi1TVQeSXAD8O3AEcFlV3TaJYzGGqZ01Yj300z62Yz30c1X3MVW18FaSpDXFO1QlqUGGuyQ1aM2E+0KPM0jyqCRXde9/Icn08lfZzwh9fF6SLyc5kOTslahxHEbo52uSfD3JrUluTHLIa3lXqxH6eH6SryXZkeTza/EO7lEfMZLk7CSVZNVeNng4I3yXr0iyv/sudyT505Wo82GqatW/mDspeydwAvBI4KvASfO2+Qvgvd3yOcBVK133BPo4DTwDeD9w9krXPMF+Ph94dLf8541+l48dWD4T+ORK1z3uPnbbHQXcBNwMzKx03RP6Ll8BvGula53/Wisj958+zqCq/g84+DiDQWcBV3TLHwVOTTLsZqrVasE+VtXuqroV+MlKFDgmo/TzM1X1QLd6M3P3Sawlo/TxhwOrj2HeTX5rwCj/TQL8HfB24H+Xs7gxGrWfq85aCfdhjzM45lDbVNUB4H7gictS3XiM0scWLLaf5wGfmGhF4zdSH5O8OsmdzIXfXy5TbeOyYB+TPBM4rqo+vpyFjdmov69/0E0jfjTJcUPeX3ZrJdwXfJzBiNusZmu9/lGN3M8kfwzMAO+YaEXjN1Ifq+rdVXUi8DrgbyZe1Xgdto9JHgFcDLx22SqajFG+y38DpqvqGcCn+dkMwopaK+E+yuMMfrpNkiOBXwbuW5bqxmO9PLJhpH4meQHwRuDMqnpwmWobl8V+lx8CXjTRisZvoT4eBTwd+GyS3cCzgWvX4EnVBb/Lqrp34Hf0UuBZy1TbYa2VcB/lcQbXAud2y2cD/1Hd2Y41Yr08smHBfnb/O//PzAX7vhWosa9R+rhpYPUM4FvLWN84HLaPVXV/VW2oqumqmmbu3MmZVTW7MuUu2Sjf5dEDq2cCty9jfYe20md0F3HW+nTgm8yduX5j1/YW5n5hAH4R+AiwC/gicMJK1zyBPv4GcyOJ/wHuBW5b6Zon1M9PA/cAO7rXtStd8wT6+I/AbV3/PgM8baVrHncf5237Wdbg1TIjfpd/332XX+2+y19d6ZqryscPSFKL1sq0jCRpEQx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1KD/B/tZyXDyrYd0AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "_=plt.hist(preds_df3[(preds_df3['actual']==1) & (preds_df3['pred_prob']>min_p)]['cont_error'], bins=50)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD4CAYAAAAAczaOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAARVklEQVR4nO3dfYzlVX3H8fenbNFqtag7WNxdOmhWDRIb7UhJTY2KTUAsa1I00KqLbrNRUdpqo1ib0rQlxdrUarS2q1AgMTzUmrKtT1WEEpOCHRAfgKgrUhhBdhTFVlPo6rd/zA87rrPcO/d37zyd9yvZzP2d37n3fk/u7mfOnt/DTVUhSWrDT612AZKklWPoS1JDDH1JaoihL0kNMfQlqSGbVrsAgM2bN9f09PRqlyFJ68oNN9zwzaqaWs5z1kToT09PMzs7u9plSNK6kuQ/l/scl3ckqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0JakhA6/ITXIh8CJgf1Udt6j99cDrgAPAh6vqTV37W4BdwA+As6vq45Mo/EHT53z4kPtuP/+USb61JK07w9yG4SLg3cAlDzYkeR6wA3h6Vd2f5Miu/VjgdOBpwBOATyZ5clX9YNyFS5KWb+DyTlVdC9x7UPNrgPOr6v6uz/6ufQdwWVXdX1VfA/YBx4+xXklSD6Ou6T8Z+NUk1yf5tyTP6tq3AHcu6jfXtUmS1oBR77K5CXgMcALwLOCKJE8EskTfJb95PcluYDfA0UcfPWIZkqTlGHWmPwd8qBZ8BvghsLlr37ao31bgrqVeoKr2VNVMVc1MTS3rdtCSpBGNGvr/BDwfIMmTgcOBbwJ7gdOTPCzJMcB24DPjKFSS1N8wp2xeCjwX2JxkDjgXuBC4MMkXgQeAnVVVwM1JrgBuYeFUzrM8c0eS1o6BoV9VZxxi18sO0f884Lw+RUmSJsMrciWpIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDDH1JaoihL0kNMfQlqSGGviQ1ZNRbK68Lh/oqRb9GUVKrnOlLUkMMfUlqiKEvSQ0x9CWpIYa+JDXE0JekhgwM/SQXJtnffTXiwft+P0kl2dxtJ8m7kuxL8vkkz5xE0ZKk0Qwz078IOOngxiTbgF8D7ljUfDILX4a+HdgNvLd/iZKkcRkY+lV1LXDvErveAbwJqEVtO4BLasF1wBFJjhpLpZKk3kZa009yKvD1qvrcQbu2AHcu2p7r2pZ6jd1JZpPMzs/Pj1KGJGmZlh36SR4BvBX4o6V2L9FWS7RRVXuqaqaqZqamppZbhiRpBKPce+dJwDHA55IAbAVuTHI8CzP7bYv6bgXu6lukJGk8lj3Tr6ovVNWRVTVdVdMsBP0zq+obwF7gFd1ZPCcA91XV3eMtWZI0qmFO2bwU+HfgKUnmkux6iO4fAW4D9gHvA147liolSWMxcHmnqs4YsH960eMCzupfliRpErwiV5IaYuhLUkMMfUlqiKEvSQ0x9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDDH1JaoihL0kNMfQlqSGGviQ1xNCXpIYY+pLUkIFfopLkQuBFwP6qOq5rezvw68ADwFeBV1bVd7p9bwF2AT8Azq6qj0+o9pFNn/PhJdtvP/+UFa5EklbWMDP9i4CTDmr7BHBcVT0d+DLwFoAkxwKnA0/rnvM3SQ4bW7WSpF4Ghn5VXQvce1Dbv1bVgW7zOmBr93gHcFlV3V9VX2Phu3KPH2O9kqQexrGm/yrgo93jLcCdi/bNdW0/IcnuJLNJZufn58dQhiRpkF6hn+StwAHgAw82LdGtlnpuVe2pqpmqmpmamupThiRpSAMP5B5Kkp0sHOA9saoeDPY5YNuibluBu0YvT5I0TiPN9JOcBLwZOLWqvr9o117g9CQPS3IMsB34TP8yJUnjMMwpm5cCzwU2J5kDzmXhbJ2HAZ9IAnBdVb26qm5OcgVwCwvLPmdV1Q8mVbwkaXkGhn5VnbFE8wUP0f884Lw+RUmSJsMrciWpIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDDH1JaoihL0kNMfQlqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0JakhA0M/yYVJ9if54qK2xyb5RJKvdD8f07UnybuS7Evy+STPnGTxkqTlGWamfxFw0kFt5wBXVdV24KpuG+BkFr4MfTuwG3jveMqUJI3DwNCvqmuBew9q3gFc3D2+GHjxovZLasF1wBFJjhpXsZKkfkZd0398Vd0N0P08smvfAty5qN9c1/YTkuxOMptkdn5+fsQyJEnLMe4DuVmirZbqWFV7qmqmqmampqbGXIYkaSmjhv49Dy7bdD/3d+1zwLZF/bYCd41eniRpnEYN/b3Azu7xTuDKRe2v6M7iOQG478FlIEnS6ts0qEOSS4HnApuTzAHnAucDVyTZBdwBvKTr/hHghcA+4PvAKydQsyRpRANDv6rOOMSuE5foW8BZfYuSJE2GV+RKUkMMfUlqiKEvSQ0x9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDDH1JaoihL0kNMfQlqSGGviQ1xNCXpIYY+pLUEENfkhrSK/ST/F6Sm5N8McmlSR6e5Jgk1yf5SpLLkxw+rmIlSf2MHPpJtgBnAzNVdRxwGHA68DbgHVW1Hfg2sGschUqS+uu7vLMJ+Jkkm4BHAHcDzwc+2O2/GHhxz/eQJI3JyKFfVV8H/pKFL0a/G7gPuAH4TlUd6LrNAVuWen6S3Ulmk8zOz8+PWoYkaRn6LO88BtgBHAM8AXgkcPISXWup51fVnqqaqaqZqampUcuQJC1Dn+WdFwBfq6r5qvpf4EPArwBHdMs9AFuBu3rWKEkakz6hfwdwQpJHJAlwInALcDVwWtdnJ3BlvxIlSePSZ03/ehYO2N4IfKF7rT3Am4E3JNkHPA64YAx1SpLGYNPgLodWVecC5x7UfBtwfJ/XlSRNhlfkSlJDDH1JaoihL0kN6bWm34rpcz68ZPvt55+ywpVIUj/O9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGuLZO5ooz3yS1hZn+pLUEENfkhri8s465JKJpFEZ+oscKkwlaaNweUeSGmLoS1JDei3vJDkCeD9wHAtfgP4q4EvA5cA0cDvw0qr6dq8qtea5NCatD31n+u8EPlZVTwV+EbgVOAe4qqq2A1d125KkNWDkmX6SRwPPAc4EqKoHgAeS7ACe23W7GLiGhe/N1QbgjF5a3/rM9J8IzAN/n+SzSd6f5JHA46vqboDu55FjqFOSNAZ9Qn8T8EzgvVX1DOB7LGMpJ8nuJLNJZufn53uUIUkaVp/QnwPmqur6bvuDLPwSuCfJUQDdz/1LPbmq9lTVTFXNTE1N9ShDkjSskUO/qr4B3JnkKV3TicAtwF5gZ9e2E7iyV4WSpLHpe0Xu64EPJDkcuA14JQu/SK5Isgu4A3hJz/eQJI1Jr9CvqpuAmSV2ndjndSVJk+G9d7QkT82UNiZvwyBJDTH0Jakhhr4kNcTQl6SGeCC3AQ91UNZv25La4kxfkhpi6EtSQwx9SWqIa/prmBdISRo3Q79x/mKR2mLobyAGuKRBXNOXpIYY+pLUEENfkhrimr5WxaGOP3iFsDRZzvQlqSG9Z/pJDgNmga9X1YuSHANcBjwWuBF4eVU90Pd9pKUs94wl/yeh1o1jpv87wK2Ltt8GvKOqtgPfBnaN4T0kSWPQa6afZCtwCnAe8IYkAZ4P/GbX5WLgj4H39nmfjcJ1bEmrre/yzl8DbwIe1W0/DvhOVR3otueALUs9McluYDfA0Ucf3bMMbXReeCaNx8jLO0leBOyvqhsWNy/RtZZ6flXtqaqZqpqZmpoatQxJ0jL0mek/Gzg1yQuBhwOPZmHmf0SSTd1sfytwV/8yJUnjMPJMv6reUlVbq2oaOB34VFX9FnA1cFrXbSdwZe8qJUljMYmLs94MXJbkz4DPAhdM4D2ksVvugXYPzGs9GkvoV9U1wDXd49uA48fxumud/+jHb9IHbD0grNZ5Ra4kNcTQl6SGGPqS1BBDX5IaYuhLUkMMfUlqiKEvSQ3xm7MmYLnngnvuuKSV4kxfkhpi6EtSQ1zekQZw+U0biTN9SWqIoS9JDTH0Jakhhr4kNcTQl6SGjHz2TpJtwCXAzwM/BPZU1TuTPBa4HJgGbgdeWlXf7l+qpIfil/poGH1O2TwAvLGqbkzyKOCGJJ8AzgSuqqrzk5wDnMPCVyhKTRhX+BrimoQ+X4x+d1Xd2D3+L+BWYAuwA7i463Yx8OK+RUqSxmMsa/pJpoFnANcDj6+qu2HhFwNw5CGeszvJbJLZ+fn5cZQhSRqgd+gn+VngH4HfrarvDvu8qtpTVTNVNTM1NdW3DEnSEHqFfpKfZiHwP1BVH+qa70lyVLf/KGB/vxIlSePS5+ydABcAt1bVXy3atRfYCZzf/byyV4XSBuGBWa0Ffc7eeTbwcuALSW7q2v6AhbC/Isku4A7gJf1KlDa21fr+BX8JtWnk0K+qTwM5xO4TR31dSdLkeEWuJDXE0Jekhhj6ktQQQ1+SGmLoS1JDDH1JaoihL0kN6XNxliQBXui1nhj6koa2WlcD+0tlfFzekaSGGPqS1BCXdyT9mHEt4WhtcqYvSQ1xpi9tcKs5c19r/2tYbj0b8UCxM31JaogzfUnr1lr7n8R6YOhLWjMM8cmbWOgnOQl4J3AY8P6qOn9S7yVJa91aucBsImv6SQ4D3gOcDBwLnJHk2Em8lyRpeJOa6R8P7Kuq2wCSXAbsAG6Z0PtJ0tiNsty01s/4mVTobwHuXLQ9B/zy4g5JdgO7u83/TvKlEd9rM/DNEZ+7EbQ8/pbHDo5/TY4/b5ts/86DY/+F5T5xUqGfJdrqxzaq9gB7er9RMltVM31fZ71qefwtjx0cf8vj7zP2SZ2nPwdsW7S9FbhrQu8lSRrSpEL/P4DtSY5JcjhwOrB3Qu8lSRrSRJZ3qupAktcBH2fhlM0Lq+rmSbwXY1giWudaHn/LYwfH3/L4Rx57qmpwL0nShuC9dySpIYa+JDVk3YR+kpOSfCnJviTnLLH/YUku7/Zfn2R65aucnCHG/5wkNyY5kOS01ahxUoYY+xuS3JLk80muSrLsc5fXsiHG/+okX0hyU5JPb6Sr3weNfVG/05JUkg11CucQn/2ZSea7z/6mJL898EWras3/YeFg8FeBJwKHA58Djj2oz2uBv+0enw5cvtp1r/D4p4GnA5cAp612zSs89ucBj+gev6bBz/7Rix6fCnxsteteqbF3/R4FXAtcB8ysdt0r/NmfCbx7Oa+7Xmb6P7qtQ1U9ADx4W4fFdgAXd48/CJyYZKmLxNajgeOvqtur6vPAD1ejwAkaZuxXV9X3u83rWLguZKMYZvzfXbT5SA66EHIdG+bfPcCfAn8B/M9KFrcChh3/sqyX0F/qtg5bDtWnqg4A9wGPW5HqJm+Y8W9Uyx37LuCjE61oZQ01/iRnJfkqC+F39grVNmkDx57kGcC2qvqXlSxshQz7d/83uqXNDybZtsT+H7NeQn/gbR2G7LNebeSxDTL02JO8DJgB3j7RilbWUOOvqvdU1ZOANwN/OPGqVsZDjj3JTwHvAN64YhWtrGE++38Gpqvq6cAn+f/VjkNaL6E/zG0dftQnySbg54B7V6S6yWv5thZDjT3JC4C3AqdW1f0rVNtKWO5nfxnw4olWtHIGjf1RwHHANUluB04A9m6gg7kDP/uq+taiv+/vA35p0Iuul9Af5rYOe4Gd3ePTgE9Vd6RjA2j5thYDx979F//vWAj8/atQ4yQNM/7tizZPAb6ygvVN0kOOvaruq6rNVTVdVdMsHM85tapmV6fcsRvmsz9q0eapwK0DX3W1j1Av40j2C4Evs3A0+61d25+w8CEDPBz4B2Af8Bngiatd8wqP/1kszAy+B3wLuHm1a17BsX8SuAe4qfuzd7VrXuHxvxO4uRv71cDTVrvmlRr7QX2vYQOdvTPkZ//n3Wf/ue6zf+qg1/Q2DJLUkPWyvCNJGgNDX5IaYuhLUkMMfUlqiKEvSQ0x9CWpIYa+JDXk/wAYwFHtGkvDqQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "_=plt.hist(preds_df3[(preds_df3['actual']==1) & (preds_df3['t1']!=preds_df3['t2']) & (preds_df3['pred_prob']>min_p)]['cont_error'], bins=50)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 50/50 Split Distribution of Residuals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD5CAYAAADcDXXiAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAO0klEQVR4nO3cf6zdd13H8eeL1YH8yjZ6IaPd7DBVMwkGvC4oCUFGDDDolrglQ8SCNQ0GHAqJK2Iyf8SkEyNqJMTqwJKQjTkxq26AY4wQ/tj0dk5gq2RlLFtZpRdhIBIZhbd/nO/YtT3tPfd87znn3s99PpKbe77f8z3n++4n5776Pp/vj1QVkqS2PGnWBUiSVp/hLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUoE3LbZDk/cBrgGNV9fxu3buB1wKPAV8E3lRVj3bPvRPYBXwPuKqqPr7cPjZv3lzbtm0b998gSRvSwYMHv1pVc8Oey3LnuSd5KfAt4INLwv0XgE9W1fEk1wJU1dVJLgSuBy4Cngt8Avixqvre6fYxPz9fCwsLK/xnSdLGluRgVc0Pe27ZaZmq+jTwtRPW/XNVHe8W7wS2do8vBW6oqu9U1ZeAwwyCXpI0Rasx5/6rwEe7x1uAh5c8d6RbJ0maol7hnuRdwHHgQ4+vGrLZ0HmfJLuTLCRZWFxc7FOGJOkEY4d7kp0MDrS+vp6YuD8CnLdks63AI8NeX1X7qmq+qubn5oYeD5AkjWmscE/ySuBqYEdVfXvJUweAK5M8OckFwHbgX/qXKUlaiVFOhbweeBmwOckR4BrgncCTgduSANxZVW+uqnuT3Ajcx2C65i3LnSkjSVp9y54KOQ2eCilJK9frVEhJ0vpjuEtSg5adc5eW2rbnlqHrH9x7yZQrkXQ6du6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGLRvuSd6f5FiSzy9Zd06S25Lc3/0+u1ufJH+R5HCSzyZ50SSLlyQNN0rn/rfAK09Ytwe4vaq2A7d3ywCvArZ3P7uB961OmZKklVg23Kvq08DXTlh9KbC/e7wfuGzJ+g/WwJ3AWUnOXa1iJUmj2TTm655TVUcBqupokmd367cADy/Z7ki37uiJb5BkN4PunvPPP3/MMrRWbNtzy9D1D+69ZMqVSILVP6CaIetq2IZVta+q5qtqfm5ubpXLkKSNbdxw/8rj0y3d72Pd+iPAeUu22wo8Mn55kqRxjBvuB4Cd3eOdwM1L1v9Kd9bMi4FvPD59I0manmXn3JNcD7wM2JzkCHANsBe4Mcku4CHgim7zW4FXA4eBbwNvmkDNkqRlLBvuVfW6Uzx18ZBtC3hL36IkSf14haokNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1KBe4Z7kt5Lcm+TzSa5P8pQkFyS5K8n9ST6c5MzVKlaSNJqxwz3JFuAqYL6qng+cAVwJXAu8p6q2A18Hdq1GoZKk0fWdltkE/HCSTcBTgaPAy4Gbuuf3A5f13IckaYXGDveq+jLwJ8BDDEL9G8BB4NGqOt5tdgTYMuz1SXYnWUiysLi4OG4ZkqQh+kzLnA1cClwAPBd4GvCqIZvWsNdX1b6qmq+q+bm5uXHLkCQN0Wda5hXAl6pqsaq+C3wE+DngrG6aBmAr8EjPGiVJK9Qn3B8CXpzkqUkCXAzcB9wBXN5tsxO4uV+JkqSV6jPnfheDA6d3A5/r3msfcDXw9iSHgWcB161CnZKkFdi0/CanVlXXANecsPoB4KI+7ytJ6scrVCWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkN6nURk7ScbXtuGbr+wb2XTLkSaWOxc5ekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QG9Qr3JGcluSnJfyQ5lORnk5yT5LYk93e/z16tYiVJo+nbuf858LGq+gngp4BDwB7g9qraDtzeLUuSpmjscE/yTOClwHUAVfVYVT0KXArs7zbbD1zWt0hJ0sr06dyfBywCH0jyb0n+JsnTgOdU1VGA7vezV6FOSdIK9An3TcCLgPdV1QuB/2EFUzBJdidZSLKwuLjYowxJ0ok29XjtEeBIVd3VLd/EINy/kuTcqjqa5Fzg2LAXV9U+YB/A/Px89ahDE7Btzy2zLkFSD2N37lX1n8DDSX68W3UxcB9wANjZrdsJ3NyrQknSivXp3AF+A/hQkjOBB4A3MfgP48Yku4CHgCt67kOStEK9wr2q7gHmhzx1cZ/3lST107dz1zpwuvnzB/deMsVKJE2Ltx+QpAYZ7pLUIMNdkhpkuEtSgzygusF5sZLUJjt3SWqQnfsacKru2dMUJY3Lzl2SGmS4S1KDDHdJapBz7g3xzBdJj7Nzl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ3yVMgp8lRFrXfeKmP9sHOXpAbZuUvqbaUdvd8AJs9wX4ec3pG0HKdlJKlBdu5S45wC2Zjs3CWpQXbukiZmpceHVutbht9W7NwlqUm9O/ckZwALwJer6jVJLgBuAM4B7gbeUFWP9d2PpI3LTnzlVqNzfxtwaMnytcB7qmo78HVg1yrsQ5K0Ar069yRbgUuAPwLeniTAy4Ff6jbZD/we8L4++5E0PevpOor1VOu09e3c/wz4beD73fKzgEer6ni3fATYMuyFSXYnWUiysLi42LMMSdJSY4d7ktcAx6rq4NLVQzatYa+vqn1VNV9V83Nzc+OWIUkaos+0zEuAHUleDTwFeCaDTv6sJJu67n0r8Ej/MtUa70UiTdbYnXtVvbOqtlbVNuBK4JNV9XrgDuDybrOdwM29q5QkrcgkznO/msHB1cMM5uCvm8A+JEmnsSpXqFbVp4BPdY8fAC5ajfeVJI3H2w/04DywpLXK2w9IUoPs3NcwL9CQNC7DfQIMZa0Hfk7b5rSMJDXIcJekBhnuktQgw12SGmS4S1KDPFtGWme8eE6jsHOXpAYZ7pLUIKdlRuDFHpLWGzt3SWqQnbvUCL9haik7d0lqkOEuSQ0y3CWpQc65a01x3lhaHYb7EgbL+uPVmlqJjfR5cVpGkhpk564mbaQOTRrGzl2SGmS4S1KDDHdJatCGnHP3rBhJrbNzl6QGjR3uSc5LckeSQ0nuTfK2bv05SW5Lcn/3++zVK1eSNIo+0zLHgXdU1d1JngEcTHIb8Ebg9qram2QPsAe4un+p0sbi9OH0tHjq7Nide1Udraq7u8f/DRwCtgCXAvu7zfYDl/UtUpK0MqtyQDXJNuCFwF3Ac6rqKAz+A0jy7FO8ZjewG+D8889fjTJOYuejlWixe9PG1fuAapKnA38P/GZVfXPU11XVvqqar6r5ubm5vmVIkpbo1bkn+SEGwf6hqvpIt/orSc7tuvZzgWN9i5RWi9/mtBKn+7ys9W90fc6WCXAdcKiq/nTJUweAnd3jncDN45cnSRpHn879JcAbgM8luadb9zvAXuDGJLuAh4Ar+pUoSVqpscO9qj4D5BRPXzzu+0qt8oCtpskrVCWpQRvy3jLSJHnQVmuBnbskNcjOXZoxO31Ngp27JDXIzl1ahme5aD2yc5ekBtm5S2NyrlxrmZ27JDXIcJekBhnuktQgw12SGuQBVUkaw1o/RdbOXZIaZLhLUoMMd0lq0Lqfc/dCEkk6mZ27JDXIcJekBq37aRlJWkvWyimSdu6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQRML9ySvTPKFJIeT7JnUfiRJJ5tIuCc5A3gv8CrgQuB1SS6cxL4kSSeb1EVMFwGHq+oBgCQ3AJcC901of5K0pk374qZJTctsAR5esnykWydJmoJJde4Zsq7+3wbJbmB3t/itJF+YUC2raTPw1VkXsQY4DgOOwxMci4EVj0Ou7bW/HznVE5MK9yPAeUuWtwKPLN2gqvYB+ya0/4lIslBV87OuY9YchwHH4QmOxcBaGodJTcv8K7A9yQVJzgSuBA5MaF+SpBNMpHOvquNJ3gp8HDgDeH9V3TuJfUmSTjaxW/5W1a3ArZN6/xlZV9NIE+Q4DDgOT3AsBtbMOKSqlt9KkrSuePsBSWqQ4T7EcrdOSPLSJHcnOZ7k8lnUOA0jjMPbk9yX5LNJbk9yytOy1rMRxuHNST6X5J4kn2n1auxRbymS5PIklWRNnDUyCSN8Jt6YZLH7TNyT5NemXmRV+bPkh8EB4C8CzwPOBP4duPCEbbYBLwA+CFw+65pnOA4/Dzy1e/zrwIdnXfeMxuGZSx7vAD4267pnMQ7dds8APg3cCczPuu4ZfibeCPzlLOu0cz/ZD26dUFWPAY/fOuEHqurBqvos8P1ZFDglo4zDHVX17W7xTgbXM7RmlHH45pLFp3HCBXuNWHYcOn8I/DHwv9MsbspGHYuZMtxP5q0TBlY6DruAj060otkYaRySvCXJFxkE21VTqm2alh2HJC8Ezquqf5pmYTMw6t/GL3ZTljclOW/I8xNluJ9s2VsnbBAjj0OSXwbmgXdPtKLZGGkcquq9VfWjwNXA7068quk77TgkeRLwHuAdU6todkb5TPwjsK2qXgB8Atg/8apOYLifbNlbJ2wQI41DklcA7wJ2VNV3plTbNK3083ADcNlEK5qN5cbhGcDzgU8leRB4MXCg0YOqo9xe5b+W/D38NfDTU6rtBwz3k3nrhIFlx6H7Gv5XDIL92AxqnIZRxmH7ksVLgPunWN+0nHYcquobVbW5qrZV1TYGx2B2VNXCbMqdqFE+E+cuWdwBHJpifcAEr1Bdr+oUt05I8gfAQlUdSPIzwD8AZwOvTfL7VfWTMyx71Y0yDgymYZ4O/F0SgIeqasfMip6AEcfhrd03mO8CXwd2zq7iyRhxHDaEEcfiqiQ7gOPA1xicPTNVXqEqSQ1yWkaSGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUoP8Dn5j3enOJvtAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "_=plt.hist(preds_df3[(preds_df3['actual']==1) & (preds_df3['t1']!=preds_df3['t2']) & (preds_df3['pred_prob']>min_p)]['5050_cont_error'], bins=50)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Last Touched Distribution of Residuals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAD4CAYAAAD1jb0+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAPQklEQVR4nO3df4xlZ13H8feHlopgSVs6bTZtxwVdflRDWxlrDQkBSk0B025iITRKFrM60aBCwMgiJiZqYtEEJLExrBQdE6CUCu6KES2bNgQjhG0pP9qlLq0FNl13S23lVwSKX/+Y0zA7O7P33Jn7Y56Z9yu5ufecObf3++TsfPrMc57z3FQVkqT2PGnaBUiS1sYAl6RGGeCS1CgDXJIaZYBLUqNOn+SHnXvuubV9+/ZJfqQkNe/OO+/8elXNLN8/0QDfvn07Bw8enORHSlLzknxlpf0OoUhSowxwSWqUAS5JjRoY4Emek+TuJY9vJHljknOS3JbkcPd89iQKliQtGhjgVXVfVV1aVZcCLwC+A3wE2AMcqKodwIFuW5I0IcMOoVwJ3F9VXwGuBRa6/QvAzlEWJkk6tWED/DXAB7rX51fVUYDu+byV3pBkPsnBJAcffvjhtVcqSTpB7wBPcgZwDfChYT6gqvZW1VxVzc3MnDQPXZK0RsP0wF8O3FVVx7rtY0m2AXTPx0ddnCRpdcPciXk9Pxw+AdgP7AJu6J73jbAuadPZvuefVtz/4A2vnHAl2ix69cCTPBW4Cvjwkt03AFclOdz97IbRlydJWk2vHnhVfQd4xrJ9j7A4K0WSNAXeiSlJjTLAJalRBrgkNcoAl6RGGeCS1CgDXJIaZYBLUqMMcElqlAEuSY2a6LfSb3WuhSFplOyBS1KjDHBJapQBLkmNMsAlqVEGuCQ1ygCXpEYZ4JLUKANckhplgEtSowxwSWqUAS5JjeoV4EnOSnJrki8lOZTk55Ock+S2JIe757PHXawk6Yf69sDfBXysqp4LXAIcAvYAB6pqB3Cg25YkTcjAAE/ydOBFwE0AVfW9qnoMuBZY6A5bAHaOq0hJ0sn6LCf7LOBh4G+SXALcCbwBOL+qjgJU1dEk56305iTzwDzA7OzsSIre6FZbNnbY411mVtKp9BlCOR34GeCvquoy4NsMMVxSVXuraq6q5mZmZtZYpiRpuT4BfgQ4UlWf7rZvZTHQjyXZBtA9Hx9PiZKklQwM8Kr6L+BrSZ7T7boSuBfYD+zq9u0C9o2lQknSivp+pdpvA+9LcgbwAPCrLIb/LUl2A18FXjWeEiVJK+kV4FV1NzC3wo+uHG05kqS+vBNTkhplgEtSowxwSWpU34uYaoA3BElbiz1wSWqUAS5JjXIIZQNzSETSqdgDl6RGGeCS1CiHUKQRGnYpYWk97IFLUqMMcElqlEMoA5zqT2Jng2gUnG2ktbIHLkmNMsAlqVEGuCQ1ygCXpEYZ4JLUKANckhplgEtSowxwSWpUrxt5kjwIfBP4AfB4Vc0lOQf4ILAdeBB4dVU9Op4ytZTrbUyf50AbwTA98JdU1aVVNddt7wEOVNUO4EC3LUmakPUMoVwLLHSvF4Cd6y9HktRX37VQCvjXJAW8u6r2AudX1VGAqjqa5LyV3phkHpgHmJ2dHUHJG4d/Rkuapr4B/sKqeqgL6duSfKnvB3Rhvxdgbm6u1lCjJGkFvYZQquqh7vk48BHgcuBYkm0A3fPxcRUpSTrZwABP8rQkZz7xGvgF4IvAfmBXd9guYN+4ipQknazPEMr5wEeSPHH8+6vqY0k+A9ySZDfwVeBV4ytTkrTcwACvqgeAS1bY/whw5TiKkiQN5jfydJxRsrVtxPPvN/VoEG+ll6RGGeCS1CiHULQpOfygrcAeuCQ1ygCXpEYZ4JLUKANckhplgEtSowxwSWqUAS5JjTLAJalR3sijpm3ENUykSbEHLkmNMsAlqVEOoUiNcZ0XPcEeuCQ1ygCXpEYZ4JLUKANckhplgEtSo3oHeJLTknw2yUe77Wcm+XSSw0k+mOSM8ZUpSVpumB74G4BDS7bfDryzqnYAjwK7R1mYJOnUegV4kguBVwLv6bYDvBS4tTtkAdg5jgIlSSvreyPPXwC/B5zZbT8DeKyqHu+2jwAXrPTGJPPAPMDs7OzaK9WaeeOHtDkN7IEn+UXgeFXduXT3CofWSu+vqr1VNVdVczMzM2ssU5K0XJ8e+AuBa5K8AngK8HQWe+RnJTm964VfCDw0vjIlScsNDPCqeivwVoAkLwZ+t6p+OcmHgOuAm4FdwL4x1imNxFZcftYhtM1rPfPA3wK8KcmXWRwTv2k0JUmS+hhqNcKqugO4o3v9AHD56EuSJPXhnZiS1CgDXJIaZYBLUqP8Rh5pixr37JRTzfhxBsxo2AOXpEYZ4JLUKIdQtKF404nUnz1wSWqUAS5JjTLAJalRBrgkNcoAl6RGGeCS1CgDXJIaZYBLUqO8kUdN2IrfpDMt3kzVDnvgktQoA1ySGuUQirRJTGuYaRKf67DOyuyBS1KjDHBJatTAIZQkTwE+AfxId/ytVfWHSZ4J3AycA9wFvLaqvjfOYiVtbs42Gk6fHvh3gZdW1SXApcDVSa4A3g68s6p2AI8Cu8dXpiRpuYEBXou+1W0+uXsU8FLg1m7/ArBzLBVKklbUaxZKktOAO4GfBG4E7gceq6rHu0OOABes8t55YB5gdnZ2vfVqhLyyL7Wt10XMqvpBVV0KXAhcDjxvpcNWee/eqpqrqrmZmZm1VypJOsFQs1Cq6jHgDuAK4KwkT/TgLwQeGm1pkqRT6TMLZQb4flU9luRHgZexeAHzduA6Fmei7AL2jbNQSdPlDJGNp88Y+DZgoRsHfxJwS1V9NMm9wM1J/gT4LHDTGOuUJC0zMMCr6vPAZSvsf4DF8XBJ0hRsubVQ/DNQmj5/D0fDW+klqVEGuCQ1ygCXpEYZ4JLUKANckhplgEtSowxwSWqUAS5JjTLAJalRBrgkNcoAl6RGbbm1ULR2p1q/Ythv8XEtDGn97IFLUqMMcElq1KYdQvFP9I3B8yCNjz1wSWqUAS5JjTLAJalRBrgkNcoAl6RGDQzwJBcluT3JoST3JHlDt/+cJLclOdw9nz3+ciVJT+jTA38ceHNVPQ+4Anh9kouBPcCBqtoBHOi2JUkTMjDAq+poVd3Vvf4mcAi4ALgWWOgOWwB2jqtISdLJhrqRJ8l24DLg08D5VXUUFkM+yXmrvGcemAeYnZ1dT62aEG++kdrQ+yJmkh8D/h54Y1V9o+/7qmpvVc1V1dzMzMxaapQkraBXgCd5Movh/b6q+nC3+1iSbd3PtwHHx1OiJGklfWahBLgJOFRV71jyo/3Aru71LmDf6MuTJK2mzxj4C4HXAl9Icne37/eBG4BbkuwGvgq8ajwlSpJWMjDAq+qTQFb58ZWjLUeS1NemXU5W0ta12kyqYb85aqPzVnpJapQBLkmNcghFI+HNP9Lk2QOXpEYZ4JLUKIdQJDVrqw/d2QOXpEYZ4JLUKANckhplgEtSowxwSWqUs1AkaRUbfU0Ve+CS1CgDXJIa5RCKpC1jow+JDMseuCQ1ygCXpEY5hCJpy2t1TRV74JLUKANckho1MMCTvDfJ8SRfXLLvnCS3JTncPZ893jIlScv16YH/LXD1sn17gANVtQM40G1LkiZoYIBX1SeA/162+1pgoXu9AOwccV2SpAHWOgvl/Ko6ClBVR5Oct9qBSeaBeYDZ2dk1ftzqWr16LEnrNfaLmFW1t6rmqmpuZmZm3B8nSVvGWgP8WJJtAN3z8dGVJEnqY61DKPuBXcAN3fO+kVUkSRvcRllTpc80wg8A/w48J8mRJLtZDO6rkhwGruq2JUkTNLAHXlXXr/KjK0dciyRpCN6JKUmNMsAlqVEGuCQ1ygCXpEYZ4JLUKANckhplgEtSowxwSWqUAS5JjfJLjSVpRCa9Roo9cElqlAEuSY0ywCWpUQa4JDXKAJekRjUzC8UvL5akE9kDl6RGGeCS1CgDXJIaZYBLUqMMcElq1LoCPMnVSe5L8uUke0ZVlCRpsDUHeJLTgBuBlwMXA9cnuXhUhUmSTm09PfDLgS9X1QNV9T3gZuDa0ZQlSRpkPTfyXAB8bcn2EeDnlh+UZB6Y7za/leS+dXzmNJ0LfH3aRYyA7dhYNkM7NkMbYIztyNvX/Z/48ZV2rifAs8K+OmlH1V5g7zo+Z0NIcrCq5qZdx3rZjo1lM7RjM7QB2mzHeoZQjgAXLdm+EHhofeVIkvpaT4B/BtiR5JlJzgBeA+wfTVmSpEHWPIRSVY8n+S3gX4DTgPdW1T0jq2zjaX4YqGM7NpbN0I7N0AZosB2pOmnYWpLUAO/ElKRGGeCS1CgDfJlBywMkeVGSu5I8nuS6adTYR492vCnJvUk+n+RAkhXnmU5Tjzb8RpIvJLk7ySc36p3AfZecSHJdkkqyIaey9Tgfr0vycHc+7k7ya9Ooc5A+5yPJq7vfj3uSvH/SNfZWVT66B4sXY+8HngWcAXwOuHjZMduB5wN/B1w37ZrX0Y6XAE/tXv8m8MFp172GNjx9yetrgI9Nu+61tKM77kzgE8CngLlp173G8/E64C+nXesI2rED+Cxwdrd93rTrXu1hD/xEA5cHqKoHq+rzwP9No8Ce+rTj9qr6Trf5KRbn8W8kfdrwjSWbT2OFG8k2gL5LTvwx8GfA/06yuCFslqUz+rTj14Ebq+pRgKo6PuEaezPAT7TS8gAXTKmW9Ri2HbuBfx5rRcPr1YYkr09yP4vh9zsTqm0YA9uR5DLgoqr66CQLG1Lff1O/1A3L3ZrkohV+Pm192vFs4NlJ/i3Jp5JcPbHqhmSAn6jX8gAN6N2OJL8CzAF/PtaKhtd3qYYbq+ongLcAfzD2qoZ3ynYkeRLwTuDNE6tobfqcj38EtlfV84GPAwtjr2p4fdpxOovDKC8Grgfek+SsMde1Jgb4iTbL8gC92pHkZcDbgGuq6rsTqq2vYc/FzcDOsVa0NoPacSbw08AdSR4ErgD2b8ALmQPPR1U9suTf0V8DL5hQbcPo8+/qCLCvqr5fVf8J3MdioG84BviJNsvyAAPb0f3Z/m4Ww3sjjvH1acPSX6pXAocnWF9fp2xHVf1PVZ1bVdurajuL1yOuqaqD0yl3VX3Ox7Ylm9cAhyZYX199fsf/gcWL/CQ5l8UhlQcmWmVf076KutEewCuA/2DxSvXbun1/xOIvFcDPsvh/6G8DjwD3TLvmNbbj48Ax4O7usX/aNa+hDe8C7unqvx34qWnXvJZ2LDv2DjbgLJSe5+NPu/Pxue58PHfaNa+xHQHeAdwLfAF4zbRrXu3hrfSS1CiHUCSpUQa4JDXKAJekRhngktQoA1ySGmWAS1KjDHBJatT/A/syQWVCXsVQAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "_=plt.hist(preds_df3[(preds_df3['actual']==1) & (preds_df3['t1']!=preds_df3['t2']) & (preds_df3['pred_prob']>min_p)]['last_touch_cont_error'], bins=50)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### First Touched Distribution of Residuals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD4CAYAAAAAczaOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAScElEQVR4nO3de5Cdd13H8fcHYkEQbCFbpiTBrU6KBganda1VR63US6HaoIKTKhIwmgErXkBtK45ldDoGdawwMMxEqE0dLNSKNnLTUlo7MKS4pfeW2lhqu7SQRS5eGIHg1z/OE+YQNt2z57J7kt/7NZPZc37P7+z5zMnms09+z3Oek6pCktSGx6x1AEnS6rH0Jakhlr4kNcTSl6SGWPqS1JB1ax0AYP369TU7O7vWMSTpqHLzzTd/uqpmVvKYqSj92dlZ5ufn1zqGJB1Vkvz7Sh/j8o4kNcTSl6SGWPqS1BBLX5IaYulLUkMsfUlqiKUvSQ1ZtvSTXJbkQJI7Dxt/ZZJ7k9yV5I/7xi9Ksr/b9uOTCC1JGs4gb866HHgjcMWhgSQ/DGwFnlNVX0xyYje+BdgGPAt4OvD+JKdU1VfGHVyStHLLln5V3Zhk9rDhVwC7quqL3ZwD3fhW4O3d+MeT7AdOBz48tsSSNKTZC9+95PgDu85Z5SRrZ9g1/VOAH0hyU5J/TvLd3fgG4KG+eQvd2NdJsjPJfJL5xcXFIWNIklZi2NJfB5wAnAH8NnBVkgBZYu6Sn8dYVburaq6q5mZmVnS9IEnSkIYt/QXgndXzEeD/gPXd+Ka+eRuBh0eLKEkal2FL/++B5wIkOQU4Dvg0sBfYluRxSU4GNgMfGUdQSdLolj2Qm+RK4ExgfZIF4GLgMuCy7jTOLwHbq6qAu5JcBdwNHATO98wdSavtSAdsNdjZO+cdYdOLjzD/EuCSUUJJkibDd+RKUkMsfUlqiKUvSQ2x9CWpIZa+JDXE0pekhlj6ktQQS1+SGmLpS1JDLH1JaoilL0kNsfQlqSGWviQ1xNKXpIZY+pLUEEtfkhpi6UtSQ5Yt/SSXJTnQfTTi4dt+K0klWd/dT5I3JNmf5PYkp00itCRpOIPs6V8OnH34YJJNwI8CD/YNP4/eh6FvBnYCbx49oiRpXJYt/aq6EfjMEpsuBX4HqL6xrcAV1bMPOD7JSWNJKkka2VBr+knOBT5RVbcdtmkD8FDf/YVubKnvsTPJfJL5xcXFYWJIklZoxaWf5AnAa4DfX2rzEmO1xBhVtbuq5qpqbmZmZqUxJElDWDfEY74NOBm4LQnARuCjSU6nt2e/qW/uRuDhUUNKksZjxXv6VXVHVZ1YVbNVNUuv6E+rqk8Ce4GXdGfxnAF8vqoeGW9kSdKwBjll80rgw8Azkywk2fEo098D3A/sB/4C+JWxpJQkjcWyyztVdd4y22f7bhdw/uixJEmT4DtyJakhlr4kNcTSl6SGWPqS1BBLX5IaMsybsyTpmDJ74buXHH9g1zmrnGTy3NOXpIZY+pLUEEtfkhrimr6ko9aR1uJ1ZO7pS1JDLH1JaoilL0kNsfQlqSGWviQ1xNKXpIYM8slZlyU5kOTOvrE/SfKxJLcn+bskx/dtuyjJ/iT3JvnxSQWXJK3cIOfpXw68Ebiib+xa4KKqOpjkdcBFwAVJtgDbgGcBTwfen+SUqvrKeGNLaonn44/Psnv6VXUj8JnDxv6pqg52d/cBG7vbW4G3V9UXq+rj9D4r9/Qx5pUkjWAca/q/CLy3u70BeKhv20I39nWS7Ewyn2R+cXFxDDEkScsZqfSTvAY4CLzt0NAS02qpx1bV7qqaq6q5mZmZUWJIkgY09LV3kmwHfgI4q6oOFfsCsKlv2kbg4eHjSZLGaag9/SRnAxcA51bVF/o27QW2JXlckpOBzcBHRo8pSRqHZff0k1wJnAmsT7IAXEzvbJ3HAdcmAdhXVS+vqruSXAXcTW/Z53zP3JGk6bFs6VfVeUsMv/VR5l8CXDJKKEnSZPiOXElqiKUvSQ3xk7Mk6Qge7Z3AD+w6ZxWTjI97+pLUEEtfkhpi6UtSQyx9SWqIpS9JDbH0Jakhlr4kNcTSl6SGWPqS1BBLX5IaYulLUkMsfUlqiKUvSQ2x9CWpIcuWfpLLkhxIcmff2FOSXJvkvu7rCd14krwhyf4ktyc5bZLhJUkrM8ie/uXA2YeNXQhcV1Wbgeu6+wDPo/dh6JuBncCbxxNTkjQOy5Z+Vd0IfOaw4a3Anu72HuAFfeNXVM8+4PgkJ40rrCRpNMOu6T+tqh4B6L6e2I1vAB7qm7fQjX2dJDuTzCeZX1xcHDKGJGklxn0gN0uM1VITq2p3Vc1V1dzMzMyYY0iSljJs6X/q0LJN9/VAN74AbOqbtxF4ePh4kqRxGrb09wLbu9vbgWv6xl/SncVzBvD5Q8tAkqS1t265CUmuBM4E1idZAC4GdgFXJdkBPAi8qJv+HuD5wH7gC8DLJpBZkjSkZUu/qs47wqazlphbwPmjhpLUptkL373WEY55viNXkhpi6UtSQyx9SWqIpS9JDbH0Jakhlr4kNcTSl6SGWPqS1BBLX5IaYulLUkMsfUlqiKUvSQ2x9CWpIZa+JDXE0pekhlj6ktQQS1+SGjJS6Sf5zSR3JbkzyZVJHp/k5CQ3JbkvyTuSHDeusJKk0Qxd+kk2AL8GzFXVs4HHAtuA1wGXVtVm4LPAjnEElSSNbtTlnXXANyZZBzwBeAR4LnB1t30P8IIRn0OSNCZDl35VfQL4U+BBemX/eeBm4HNVdbCbtgBsWOrxSXYmmU8yv7i4OGwMSdIKjLK8cwKwFTgZeDrwROB5S0ytpR5fVburaq6q5mZmZoaNIUlagVGWd34E+HhVLVbVl4F3At8HHN8t9wBsBB4eMaMkaUxGKf0HgTOSPCFJgLOAu4HrgRd2c7YD14wWUZI0LqOs6d9E74DtR4E7uu+1G7gAeFWS/cBTgbeOIackaQzWLT/lyKrqYuDiw4bvB04f5ftKOrbNXvjutY7QLN+RK0kNsfQlqSGWviQ1xNKXpIZY+pLUEEtfkhpi6UtSQyx9SWqIpS9JDbH0Jakhlr4kNcTSl6SGjHTBNUlq1ZEuGvfArnNWOcnKuKcvSQ2x9CWpIZa+JDVkpNJPcnySq5N8LMk9Sb43yVOSXJvkvu7rCeMKK0kazah7+q8H3ldV3w58J3APcCFwXVVtBq7r7kuSpsDQZ+8keTLwg8BLAarqS8CXkmwFzuym7QFuoPe5uTqKHK1nJkh6dKPs6X8rsAj8ZZJbkrwlyROBp1XVIwDd1xPHkFOSNAajlP464DTgzVV1KvA/rGApJ8nOJPNJ5hcXF0eIIUka1CilvwAsVNVN3f2r6f0S+FSSkwC6rweWenBV7a6quaqam5mZGSGGJGlQQ5d+VX0SeCjJM7uhs4C7gb3A9m5sO3DNSAklSWMz6mUYXgm8LclxwP3Ay+j9IrkqyQ7gQeBFIz6HJGlMRir9qroVmFti01mjfF9J0mR4wbUGHOn0S0nt8TIMktQQ9/SnmG+QkjRu7ulLUkPc05c0MR5Pmj7u6UtSQyx9SWqIpS9JDbH0JakhHsg9hnjQTNJy3NOXpIZY+pLUEEtfkhpi6UtSQzyQexTygK2kYVn6U+BoKnEvAicd3VzekaSGjFz6SR6b5JYk7+run5zkpiT3JXlH91GKkqQpMI49/V8H7um7/zrg0qraDHwW2DGG55AkjcFIa/pJNgLnAJcAr0oS4LnAz3VT9gCvBd48yvNI0tFi2o97jbqn/+fA7wD/191/KvC5qjrY3V8ANiz1wCQ7k8wnmV9cXBwxhiRpEEOXfpKfAA5U1c39w0tMraUeX1W7q2ququZmZmaGjSFJWoFRlne+Hzg3yfOBxwNPprfnf3ySdd3e/kbg4dFjSpLGYeg9/aq6qKo2VtUssA34QFX9PHA98MJu2nbgmpFTSpLGYhLn6V9A76Dufnpr/G+dwHNIkoYwlnfkVtUNwA3d7fuB08fxfSVJ4+U7ciWpIZa+JDXE0pekhlj6ktQQS1+SGmLpS1JDLH1JaoilL0kN8eMSV9HR9LGIko5N7ulLUkPc09dYTPsHR0jqcU9fkhpi6UtSQyx9SWqIpS9JDfFA7gR4aqakaWXpSxqZOzpHj6GXd5JsSnJ9knuS3JXk17vxpyS5Nsl93dcTxhdXkjSKUdb0DwKvrqrvAM4Azk+yBbgQuK6qNgPXdfclSVNg6NKvqkeq6qPd7f8C7gE2AFuBPd20PcALRg0pSRqPsazpJ5kFTgVuAp5WVY9A7xdDkhOP8JidwE6AZzzjGeOIoaOI7+CV1sbIp2wm+Sbgb4HfqKr/HPRxVbW7quaqam5mZmbUGJKkAYxU+km+gV7hv62q3tkNfyrJSd32k4ADo0WUJI3L0Ms7SQK8Fbinqv6sb9NeYDuwq/t6zUgJp5inqS3P10iaLqOs6X8/8AvAHUlu7cZ+l17ZX5VkB/Ag8KLRIkqSxmXo0q+qDwI5wuazhv2+kqTJ8do7ktQQS1+SGmLpS1JDLH1JaohX2RyApx1KOla4py9JDbH0JakhTS7veLEvSa1qsvQlabVNy86myzuS1BBLX5Iackwv73iqpSR9rWO69HX0mZZ1T+lYZelLGpj/ez76uaYvSQ056vf03fOQpMEd9aU/Tv4CkXSsm1jpJzkbeD3wWOAtVbVrUs8laXzc+Tm2TWRNP8ljgTcBzwO2AOcl2TKJ55IkDW5Se/qnA/ur6n6AJG8HtgJ3T+j5dIwb196np36qdZMq/Q3AQ333F4Dv6Z+QZCews7v730nunVCWQawHPr2Gz38k05hrGjPBgLnyulVI8rWm8fWaxkzQaK4hfyYPZfqWlT5wUqWfJcbqa+5U7QZ2T+j5VyTJfFXNrXWOw01jrmnMBOZaiWnMBOZaiVEyTeo8/QVgU9/9jcDDE3ouSdKAJlX6/wJsTnJykuOAbcDeCT2XJGlAE1neqaqDSX4V+Ed6p2xeVlV3TeK5xmQqlpmWMI25pjETmGslpjETmGslhs6Uqlp+liTpmOC1dySpIZa+JDWkqdJPcnaSe5PsT3LhEttfleTuJLcnuS7Jis+BnUCmlye5I8mtST64Wu9sXi5X37wXJqkkq3JK2wCv10uTLHav161JfmmtM3Vzfrb72boryV9POtMguZJc2vc6/WuSz01JrmckuT7JLd2/xedPQaZv6Trh9iQ3JNk46Uzd816W5ECSO4+wPUne0OW+Pclpy37TqmriD70Dyv8GfCtwHHAbsOWwOT8MPKG7/QrgHVOQ6cl9t88F3jcNr1U370nAjcA+YG4acgEvBd44ZT9Xm4FbgBO6+ydOQ67D5r+S3gkXa56L3kHKV3S3twAPTEGmvwG2d7efC/zVKv18/SBwGnDnEbY/H3gvvfdGnQHctNz3bGlP/6uXhqiqLwGHLg3xVVV1fVV9obu7j977C9Y603/23X0ih73Jba1ydf4Q+GPgf1ch00pyraZBMv0y8Kaq+ixAVR2Yklz9zgOunJJcBTy5u/3NTP49PoNk2gJc192+fontE1FVNwKfeZQpW4ErqmcfcHySkx7te7ZU+ktdGmLDo8zfQe836CQNlCnJ+Un+jV7B/tqEMw2UK8mpwKaqetcq5Bk4V+dnuv/qXp1k0xLbVzvTKcApST6UZF93BdpJG/jnvVvGPBn4wJTkei3w4iQLwHvo/S9krTPdBvxMd/ungCcleeqEcw1ipb3WVOkve2mIr05MXgzMAX8y0UQDZqqqN1XVtwEXAL834UywTK4kjwEuBV69Cln6DfJ6/QMwW1XPAd4P7JmCTOvoLfGcSW+P+i1Jjp+CXIdsA66uqq9MMM8hg+Q6D7i8qjbSW774q+5nbi0z/RbwQ0luAX4I+ARwcIKZBrWSv2egrdIf6NIQSX4EeA1wblV9cRoy9Xk78IKJJupZLteTgGcDNyR5gN5a4t5VOJi77OtVVf/R9/f2F8B3rXWmbs41VfXlqvo4cC+9XwJrneuQbazO0g4MlmsHcBVAVX0YeDy9C4ytWaaqeriqfrqqTqXXD1TV5yeYaVArv+TNahyMmIY/9Pa27qf339hDB2ueddicU+kd0Nk8RZk2993+SWB+GnIdNv8GVudA7iCv10l9t38K2DcFmc4G9nS319P77/hT1zpXN++ZwAN0b9Sckr/D9wIv7W5/R1diE8s3YKb1wGO625cAf7Aar1f3fLMc+UDuOXztgdyPLPv9Viv4NPyh91/Ff+2K/TXd2B/Q26uH3nLAp4Bbuz97pyDT64G7ujzXP1r5rmauw+auSukP+Hr9Ufd63da9Xt8+BZkC/Bm9z5O4A9g2Da9Vd/+1wK7VyLOC12sL8KHu7/BW4MemINMLgfu6OW8BHrdKr9WVwCPAl+nt1e8AXg68vO9n601d7jsG+XfoZRgkqSEtrelLUvMsfUlqiKUvSQ2x9CWpIZa+JDXE0pekhlj6ktSQ/wd4THM6xudx9QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "_=plt.hist(preds_df3[(preds_df3['actual']==1) & (preds_df3['t1']!=preds_df3['t2']) & (preds_df3['pred_prob']>min_p)]['first_touch_cont_error'], bins=50)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Impact of Noise on the Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Distribution of error in False Positives is left skewed as expected. \n",
    "##### The model made the right inference of \"Likely Conversion\" based solely on the inputs available to it onaverage,but was screwed from the get go due to the large, negative error that ended up with the majority of impact in determining the actual conversion status"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAQFklEQVR4nO3df4xlZX3H8fenoCb1R8DuQClgB81qSlu7mAnaGBWLtvwwIiZYto1SJV1NINFoU/mRiGljglUkMbaYNWyABFZokUAKrWyJkZiIdUBcFxfKQlcY2O6O0qANxmbh2z/mDL0sd5g7c++dYZ59v5Kbe89zzrnne3aTzz773HOek6pCktSWX1vtAiRJo2e4S1KDDHdJapDhLkkNMtwlqUGHrnYBAOvWravJycnVLkOS1pS77777p1U10W/diyLcJycnmZ6eXu0yJGlNSfKThdY5LCNJDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ16UdyhKmltmrzg1oG2233p6WOuRAey5y5JDTLcJalBhrskNcgxd0nPM+hYul68DHdJY+cPryvPYRlJapDhLkkNMtwlqUGGuyQ1aNFwT3Jskm8l2ZnkviQf79pfnWRbkge798O79iT5cpJdSbYnedO4T0KS9FyD9Nz3A5+qqt8B3gKcl+R44ALgjqpaD9zRLQOcCqzvXpuAK0ZetSTpBS0a7lW1p6ru6T7/AtgJHA2cAVzdbXY18L7u8xnANTXnLuCwJEeNvHJJ0oKWNOaeZBI4AfgecGRV7YG5fwCAI7rNjgYe7dltpms78Ls2JZlOMj07O7v0yiVJCxo43JO8ArgR+ERV/fyFNu3TVs9rqNpcVVNVNTUxMTFoGZKkAQwU7klewlywX1tV3+ia984Pt3Tv+7r2GeDYnt2PAR4fTbmSpEEsOv1AkgBXAjur6ks9q24BzgEu7d5v7mk/P8nXgTcDT84P30jSKDidweIGmVvmrcAHgR8lubdru4i5UL8hybnAI8BZ3brbgNOAXcBTwIdHWrEkaVGLhntVfYf+4+gAJ/fZvoDzhqxLkjQE71CVpAYZ7pLUIOdzl/Si4UNCRseeuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQl0JKBwkvMzy42HOXpAYZ7pLUIMNdkhpkuEtSgwx3SWrQouGeZEuSfUl29LRdn+Te7rV7/iEeSSaT/LJn3VfHWbwkqb9BLoW8CvgKcM18Q1X96fznJJcBT/Zs/1BVbRhVgZKkpRvkSUx3Jpnst657vuoHgD8abVmSpGEMO+b+NmBvVT3Y03Zckh8k+XaSty20Y5JNSaaTTM/Ozg5ZhiSp17DhvhHY2rO8B3hNVZ0AfBK4Lsmr+u1YVZuraqqqpiYmJoYsQ5LUa9nhnuRQ4P3A9fNtVfWrqvpZ9/lu4CHg9cMWKUlammF67u8C7q+qmfmGJBNJDuk+vxZYDzw8XImSpKUa5FLIrcB3gTckmUlybrfqbJ47JAPwdmB7kh8C/wR8rKqeGGXBkqTFDXK1zMYF2v+iT9uNwI3DlyVJGoZ3qEpSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktSgQR7WsSXJviQ7eto+m+SxJPd2r9N61l2YZFeSB5L8ybgKlyQtbJCe+1XAKX3aL6+qDd3rNoAkxzP3hKbf7fb5h/nH7kmSVs6i4V5VdwKDPirvDODr3YOy/xPYBZw4RH2SpGUYZsz9/CTbu2Gbw7u2o4FHe7aZ6dokSStoueF+BfA6YAOwB7isa0+fbavfFyTZlGQ6yfTs7Owyy5Ak9bOscK+qvVX1dFU9A3yN/x96mQGO7dn0GODxBb5jc1VNVdXUxMTEcsqQJC1gWeGe5KiexTOB+StpbgHOTvKyJMcB64F/H65ESdJSHbrYBkm2AicB65LMAJcAJyXZwNyQy27gowBVdV+SG4AfA/uB86rq6fGULklayKLhXlUb+zRf+QLbfw743DBFSZKG4x2qktQgw12SGrTosIwkrVWTF9w60Ha7Lz19zJWsPHvuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhrk9APSGjfoLfY6uNhzl6QGGe6S1KBFwz3JliT7kuzoaftCkvuTbE9yU5LDuvbJJL9Mcm/3+uo4i5ck9TfImPtVwFeAa3ratgEXVtX+JJ8HLgQ+3a17qKo2jLRK6SDkWLqGsWjPvaruBJ44oO32qtrfLd4FHDOG2iRJyzSKMfePAP/Ss3xckh8k+XaSty20U5JNSaaTTM/Ozo6gDEnSvKHCPcnFwH7g2q5pD/CaqjoB+CRwXZJX9du3qjZX1VRVTU1MTAxThiTpAMsO9yTnAO8B/ryqCqCqflVVP+s+3w08BLx+FIVKkga3rJuYkpzC3A+o76iqp3raJ4AnqurpJK8F1gMPj6RSqRH+UKqVsGi4J9kKnASsSzIDXMLc1TEvA7YlAbirqj4GvB34myT7gaeBj1XVE32/WJI0NouGe1Vt7NN85QLb3gjcOGxRkqTheIeqJDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDRoo3JNsSbIvyY6etlcn2Zbkwe798K49Sb6cZFeS7UneNK7iJUn9Ddpzvwo45YC2C4A7qmo9cEe3DHAqc4/XWw9sAq4YvkxJ0lIMFO5VdSdw4OPyzgCu7j5fDbyvp/2amnMXcFiSo0ZRrCRpMMOMuR9ZVXsAuvcjuvajgUd7tpvp2p4jyaYk00mmZ2dnhyhDknSgcfygmj5t9byGqs1VNVVVUxMTE2MoQ5IOXsOE+9754ZbufV/XPgMc27PdMcDjQxxHkrREw4T7LcA53edzgJt72j/UXTXzFuDJ+eEbSdLKOHSQjZJsBU4C1iWZAS4BLgVuSHIu8AhwVrf5bcBpwC7gKeDDI65ZkrSIgcK9qjYusOrkPtsWcN4wRUmShuMdqpLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBA10KKWlxkxfcutolSM+y5y5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoO8zl1ahNevay1adrgneQNwfU/Ta4HPAIcBfwnMP/X6oqq6bdkVSpKWbNnhXlUPABsAkhwCPAbcxNyTly6vqi+OpEJJ0pKNasz9ZOChqvrJiL5PkjSEUYX72cDWnuXzk2xPsiXJ4f12SLIpyXSS6dnZ2X6bSJKWaehwT/JS4L3AP3ZNVwCvY27IZg9wWb/9qmpzVU1V1dTExMSwZUiSeoyi534qcE9V7QWoqr1V9XRVPQN8DThxBMeQJC3BKMJ9Iz1DMkmO6ll3JrBjBMeQJC3BUNe5J/l14N3AR3ua/y7JBqCA3QeskyStgKHCvaqeAn7jgLYPDlWRJGloTj8gSQ0y3CWpQYa7JDXIcJekBhnuktQgp/zVQcupfNUye+6S1CB77pIOekv5X9zuS08fYyWjY89dkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGDX2de5LdwC+Ap4H9VTWV5NXA9cAkcw/s+EBV/fewx5IkDWZUPfd3VtWGqprqli8A7qiq9cAd3bIkaYWMa1jmDODq7vPVwPvGdBxJUh+jCPcCbk9yd5JNXduRVbUHoHs/4sCdkmxKMp1kenZ2dgRlSJLmjWJumbdW1eNJjgC2Jbl/kJ2qajOwGWBqaqpGUIckqTN0z72qHu/e9wE3AScCe5McBdC97xv2OJKkwQ0V7klenuSV85+BPwZ2ALcA53SbnQPcPMxxJElLM+ywzJHATUnmv+u6qvrXJN8HbkhyLvAIcNaQx5EG4gM4pDlDhXtVPQz8QZ/2nwEnD/PdkqTl8w5VSWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lq0ChmhZTGzmkFpKWx5y5JDTLcJalBhrskNchwl6QGGe6S1KBlh3uSY5N8K8nOJPcl+XjX/tkkjyW5t3udNrpyJUmDGOZSyP3Ap6rqnu5Re3cn2datu7yqvjh8eWqdlzhK47HscK+qPcCe7vMvkuwEjh5VYZKk5RvJmHuSSeAE4Htd0/lJtifZkuTwURxDkjS4ocM9ySuAG4FPVNXPgSuA1wEbmOvZX7bAfpuSTCeZnp2dHbYMSVKPocI9yUuYC/Zrq+obAFW1t6qerqpngK8BJ/bbt6o2V9VUVU1NTEwMU4Yk6QDDXC0T4EpgZ1V9qaf9qJ7NzgR2LL88SdJyDHO1zFuBDwI/SnJv13YRsDHJBqCA3cBHh6pQkrRkw1wt8x0gfVbdtvxyJEmj4B2qktQgw12SGuTDOjQW3nkqrS7DXZKWYNCOy+5LTx9zJS/MYRlJapA9dy2Jwy3S2mDPXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgb2IS4M1JUmvsuUtSg8YW7klOSfJAkl1JLhjXcSRJzzeWYZkkhwB/D7wbmAG+n+SWqvrxOI53sHEIRXrxW+3ZI8c15n4isKuqHgZI8nXgDGAs4T7qsBv0D3vUf3mGtqRRGVe4Hw082rM8A7y5d4Mkm4BN3eL/JHlgBMddB/x02C/J50dQyRi/r8dIzncN8XzbdjCd77PnOmQ+/PZCK8YV7v0enF3PWajaDGwe6UGT6aqaGuV3vph5vm3zfNu1Euc6rh9UZ4Bje5aPAR4f07EkSQcYV7h/H1if5LgkLwXOBm4Z07EkSQcYy7BMVe1Pcj7wTeAQYEtV3TeOYx1gpMM8a4Dn2zbPt11jP9dU1eJbSZLWFO9QlaQGGe6S1KBmwz3JXyWpJOtWu5ZxSvK3SbYnuTfJ7Ul+a7VrGqckX0hyf3fONyU5bLVrGpckZyW5L8kzSZq9RPBgmqokyZYk+5LsGPexmgz3JMcyN/XBI6tdywr4QlW9sao2AP8MfGa1CxqzbcDvVdUbgf8ALlzlesZpB/B+4M7VLmRceqYqORU4HtiY5PjVrWqsrgJOWYkDNRnuwOXAX3PAjVMtqqqf9yy+nMbPuapur6r93eJdzN1D0aSq2llVo7hz+8Xs2alKqup/gfmpSppUVXcCT6zEsZqbzz3Je4HHquqHSb8bZduT5HPAh4AngXeucjkr6SPA9atdhIay6FQlWp41Ge5J/g34zT6rLgYuAv54ZSsarxc636q6uaouBi5OciFwPnDJihY4Youdb7fNxcB+4NqVrG3UBjnXxi06VYmWZ02Ge1W9q197kt8HjgPme+3HAPckObGq/msFSxyphc63j+uAW1nj4b7Y+SY5B3gPcHKt8Rs1lvB32yqnKhmTNRnuC6mqHwFHzC8n2Q1MVVWzM80lWV9VD3aL7wXuX816xi3JKcCngXdU1VOrXY+G9uxUJcBjzE1V8merW1IbWv1B9WByaZIdSbYzNxz18dUuaMy+ArwS2NZd/vnV1S5oXJKcmWQG+EPg1iTfXO2aRq37cXx+qpKdwA0rNFXJqkiyFfgu8IYkM0nOHdux1vj/aiVJfdhzl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQf8H9Hk6OCUPOGcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "_=plt.hist(preds_df2[(preds_df2['actual']==0) & (preds_df2['pred']==True)]['epsilon'], bins=30)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Distribution of error in True Positives is more symmetrical and centered closer to zero\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAOa0lEQVR4nO3dfYxl9V3H8fenLEgprTzsgCtghybERIkKTpBK0pBSEwoNSyIYjGkXgtnEp1JrIit/SPSvJTF98CGQtaBbQ1oIJbIC1SCFNP7RtQOl8rCtrIiwsrLTB6C1xrr69Y97aKfDHfbOnDtz7/x4v5LJ3HvOufd89uyez577u+eem6pCktSWN006gCRp/Cx3SWqQ5S5JDbLcJalBlrskNWjTpAMAbN68uWZnZycdQ5I2lEceeeRrVTUzbN4Ryz3JbcD7gENVdXY37STgDmAWeBb4par6ZpIAHwcuAb4DXF1Vjx5pHbOzs8zPz4/2p5EkAZDk35abN8qwzF8CFy+ZtgN4sKrOAh7s7gO8Fzir+9kO3LzSsJKk/o5Y7lX1eeAbSyZvBXZ3t3cDly+a/ska+AJwQpIt4worSRrNat9QPbWqDgJ0v0/ppp8GPL9ouQPdtNdIsj3JfJL5hYWFVcaQJA0z7rNlMmTa0OsbVNWuqpqrqrmZmaHvB0iSVmm15f7iq8Mt3e9D3fQDwBmLljsdeGH18SRJq7Hact8DbOtubwPuWTT9Axk4H3j51eEbSdL6GeVUyE8BFwKbkxwAbgR2AncmuRZ4DriyW/x+BqdB7mdwKuQ1a5BZknQERyz3qvrlZWZdNGTZAn6jbyhJUj9efkCSGjQVlx/Q2pjdcd9Iyz2789I1TiJpvXnkLkkNstwlqUGWuyQ1yHKXpAZZ7pLUIMtdkhpkuUtSgyx3SWqQ5S5JDbLcJalBlrskNchyl6QGWe6S1CDLXZIaZLlLUoMsd0lqkOUuSQ2y3CWpQZa7JDXIcpekBlnuktQgy12SGrRp0gG0crM77pt0BElTziN3SWqQ5S5JDbLcJalBjrlrRWP4z+68dA2TSBoXj9wlqUGWuyQ1yHKXpAZZ7pLUIN9Q1YqM+uarb7xKk9XryD3Jbyd5MskTST6V5NgkZybZm+TpJHckOWZcYSVJo1l1uSc5DfggMFdVZwNHAVcBNwEfraqzgG8C144jqCRpdH3H3DcBb06yCTgOOAi8G7irm78buLznOiRJK7Tqcq+qfwf+CHiOQam/DDwCvFRVh7vFDgCnDXt8ku1J5pPMLywsrDaGJGmIPsMyJwJbgTOBHwXeArx3yKI17PFVtauq5qpqbmZmZrUxJElD9BmWeQ/wr1W1UFX/A9wN/DxwQjdMA3A68ELPjJKkFepT7s8B5yc5LkmAi4CngIeAK7pltgH39IsoSVqpPmPuexm8cfoo8Hj3XLuA64EPJ9kPnAzcOoackqQV6PUhpqq6EbhxyeRngPP6PK8kqR8vPyBJDbLcJalBlrskNchyl6QGWe6S1CDLXZIaZLlLUoMsd0lqkOUuSQ2y3CWpQZa7JDXIcpekBlnuktQgy12SGmS5S1KDLHdJapDlLkkNstwlqUGWuyQ1yHKXpAZZ7pLUIMtdkhpkuUtSgyx3SWqQ5S5JDbLcJalBlrskNchyl6QGWe6S1CDLXZIaZLlLUoMsd0lq0KZJB9D3ze64b9IRJDXCI3dJalCvck9yQpK7knwlyb4k70xyUpIHkjzd/T5xXGElSaPpOyzzceBvq+qKJMcAxwE3AA9W1c4kO4AdwPU916MNZtQhpmd3XrrGSaQ3plUfuSd5G/Au4FaAqvpuVb0EbAV2d4vtBi7vG1KStDJ9hmXeASwAf5HkS0k+keQtwKlVdRCg+33KsAcn2Z5kPsn8wsJCjxiSpKX6lPsm4Fzg5qo6B/hPBkMwI6mqXVU1V1VzMzMzPWJIkpbqU+4HgANVtbe7fxeDsn8xyRaA7vehfhElSSu16nKvqv8Ank/y492ki4CngD3Atm7aNuCeXgklSSvW92yZ3wJu786UeQa4hsF/GHcmuRZ4Driy5zokSSvUq9yr6jFgbsisi/o8rySpHz+hKkkNstwlqUGWuyQ1yHKXpAZZ7pLUIMtdkhpkuUtSgyx3SWqQ5S5JDbLcJalBlrskNchyl6QGWe6S1CDLXZIaZLlLUoMsd0lqkOUuSQ2y3CWpQZa7JDWo7xdkS73M7rhvpOWe3XnpGieR2uKRuyQ1yHKXpAZZ7pLUIMtdkhpkuUtSgyx3SWqQ5S5JDbLcJalBfohpHYz6QR1JGheP3CWpQZa7JDXIcpekBlnuktSg3uWe5KgkX0pyb3f/zCR7kzyd5I4kx/SPKUlaiXGcLXMdsA94W3f/JuCjVfXpJLcA1wI3j2E9egPz0sDSyvQ6ck9yOnAp8InufoB3A3d1i+wGLu+zDknSyvUdlvkY8LvA/3X3TwZeqqrD3f0DwGnDHphke5L5JPMLCws9Y0iSFlt1uSd5H3Coqh5ZPHnIojXs8VW1q6rmqmpuZmZmtTEkSUP0GXO/ALgsySXAsQzG3D8GnJBkU3f0fjrwQv+YkqSVWPWRe1X9XlWdXlWzwFXA56rqV4CHgCu6xbYB9/ROKUlakbU4z/164MNJ9jMYg791DdYhSXodY7lwWFU9DDzc3X4GOG8czytJWh0/oSpJDbLcJalBlrskNchyl6QGWe6S1CC/Zk9NGfdXGnohMm1UHrlLUoMsd0lqkOUuSQ2y3CWpQZa7JDXIcpekBlnuktQgy12SGmS5S1KDLHdJapCXH5Bex0ouZ+ClCjRNPHKXpAZZ7pLUIMtdkhpkuUtSgyx3SWqQZ8tI62zUM3A8+0Z9eOQuSQ3yyL2HcX+lmySNi0fuktQgy12SGmS5S1KDLHdJapDlLkkNstwlqUGWuyQ1yHKXpAZZ7pLUoFWXe5IzkjyUZF+SJ5Nc100/KckDSZ7ufp84vriSpFH0ufzAYeB3qurRJG8FHknyAHA18GBV7UyyA9gBXN8/qjTdvByFpsmqj9yr6mBVPdrd/hawDzgN2Ars7hbbDVzeN6QkaWXGMuaeZBY4B9gLnFpVB2HwHwBwyjKP2Z5kPsn8wsLCOGJIkjq9yz3J8cBngA9V1SujPq6qdlXVXFXNzczM9I0hSVqkV7knOZpBsd9eVXd3k19MsqWbvwU41C+iJGml+pwtE+BWYF9VfWTRrD3Atu72NuCe1ceTJK1Gn7NlLgDeDzye5LFu2g3ATuDOJNcCzwFX9osoSVqpVZd7Vf0DkGVmX7Ta55Uk9ecnVCWpQZa7JDXIcpekBlnuktQgy12SGmS5S1KDLHdJapDlLkkNstwlqUGWuyQ1yHKXpAb1uXBYs/y6NEkbnUfuktQgj9ylKTXqK8hnd166xkm0EXnkLkkNstwlqUGWuyQ1yHKXpAa9Yd5Q9fRGSW8kHrlLUoMsd0lqkOUuSQ16w4y5S60a9/tJfiiqDR65S1KDLHdJapDlLkkNcsxd0g/wgmVt8MhdkhpkuUtSgyx3SWqQ5S5JDbLcJalBlrskNWjDnwrppXwl6bXW5Mg9ycVJvppkf5Ida7EOSdLyxn7knuQo4M+AXwAOAF9Msqeqnhr3uiRNzkpeNY/6gadJvRKfZL61+jDYWhy5nwfsr6pnquq7wKeBrWuwHknSMtZizP004PlF9w8AP7d0oSTbge3d3W8n+eqYc2wGvjbm5xynac8H059x2vPB9Gdcl3y5adUPnfZ80DNjz3W/fbkZa1HuGTKtXjOhahewaw3WPwiRzFfV3Fo9f1/Tng+mP+O054Ppz2i+/qY141oMyxwAzlh0/3TghTVYjyRpGWtR7l8EzkpyZpJjgKuAPWuwHknSMsY+LFNVh5P8JvB3wFHAbVX15LjXM4I1G/IZk2nPB9OfcdrzwfRnNF9/U5kxVa8ZDpckbXBefkCSGmS5S1KDNny5H+lSB0l+KMkd3fy9SWanLN/VSRaSPNb9/Oo657styaEkTywzP0n+uMv/T0nOnbJ8FyZ5edH2+/11zndGkoeS7EvyZJLrhiwz6W04SsaJbcckxyb5xyRf7vL9wZBlJr0fj5Jxovvya1TVhv1h8IbtvwDvAI4Bvgz8xJJlfh24pbt9FXDHlOW7GvjTCW7DdwHnAk8sM/8S4LMMPr9wPrB3yvJdCNw7we23BTi3u/1W4J+H/B1PehuOknFi27HbLsd3t48G9gLnL1lmYvvxCjJOdF9e+rPRj9xHudTBVmB3d/su4KIkwz5oNal8E1VVnwe+8TqLbAU+WQNfAE5IsmV90o2Ub6Kq6mBVPdrd/hawj8GntBeb9DYcJePEdNvl293do7ufpWd6THI/HjXjVNno5T7sUgdL/9F+b5mqOgy8DJy8LulGywfwi93L9buSnDFk/iSN+meYpHd2L5c/m+QnJxWiGyo4h8FR3WJTsw1fJyNMcDsmOSrJY8Ah4IGqWnYbTmA/HjUjTNG+vNHLfZRLHYx0OYQ1Msq6/waYraqfAv6e7x+dTItJbr9RPAq8vap+GvgT4K8nESLJ8cBngA9V1StLZw95yLpvwyNknOh2rKr/raqfYfCJ9vOSnL1kkYlvwxEyTtW+vNHLfZRLHXxvmSSbgB9m/V7mHzFfVX29qv67u/vnwM+uU7ZRTfXlJKrqlVdfLlfV/cDRSTavZ4YkRzMozdur6u4hi0x8Gx4p4zRsx27dLwEPAxcvmTXJ/fgHLJdx2vbljV7uo1zqYA+wrbt9BfC56t79mIZ8S8ZeL2MwHjpN9gAf6M74OB94uaoOTjrUq5L8yKtjr0nOY/Bv+uvruP4AtwL7quojyyw20W04SsZJbsckM0lO6G6/GXgP8JUli01yPx4p47Ttyxv6a/ZqmUsdJPlDYL6q9jD4R/1XSfYz+J/+qinL98EklwGHu3xXr1c+gCSfYnCmxOYkB4AbGbxZRFXdAtzP4GyP/cB3gGumLN8VwK8lOQz8F3DVeu70wAXA+4HHu/FYgBuAH1uUcaLbcMSMk9yOW4DdGXzRz5uAO6vq3mnZj1eQcaL78lJefkCSGrTRh2UkSUNY7pLUIMtdkhpkuUtSgyx3SWqQ5S5JDbLcJalB/w+scdLsv5LbrAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "_=plt.hist(preds_df2[(preds_df2['actual']==1) & (preds_df2['pred']==True)]['epsilon'], bins=30)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Distribution of error in False Negatives is right skewed as expected\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAARIklEQVR4nO3df4xlZX3H8feniPYHtqA70A2wLho0UaMLTpCGaGjRFsGAtmohDYKlXWkl1dQ/RJqINTHZtqKptcUshQANIlREt4qtlFqJSUFnERFc0IWirGzYESpgMCSL3/4xZ/Qy3Nm5M/fe+fHwfiU395znnHPv9+GwnznzzPmRqkKS1JZfWukCJEmjZ7hLUoMMd0lqkOEuSQ0y3CWpQc9a6QIA1q1bVxs3blzpMiRpTdm+ffuPqmqi37JVEe4bN25kampqpcuQpDUlyffnW+awjCQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBC4Z7ksOTfCXJjiR3Jnl31/68JDck+V73flDXniQfT7Izye1Jjh53JyRJTzXIFap7gfdW1a1JngtsT3IDcBZwY1VtSXIecB7wPuANwJHd69XARd27VrGN531xoPXu23LymCuRNAoLHrlX1e6qurWbfgzYARwKnApc3q12OfCmbvpU4IqacTNwYJL1I69ckjSvRY25J9kIHAXcAhxSVbth5gcAcHC32qHA/T2b7era5n7W5iRTSaamp6cXX7kkaV4Dh3uSA4BrgfdU1aP7WrVP29Me1FpVW6tqsqomJyb63tRMkrREA4V7kv2ZCfYrq+qzXfODs8Mt3fuern0XcHjP5ocBD4ymXEnSIAY5WybAJcCOqvpoz6JtwJnd9JnA53va396dNXMs8Mjs8I0kaXkMcrbMccAZwLeT3Na1nQ9sAa5JcjbwA+Ct3bLrgZOAncDjwDtGWrEkaUELhntVfY3+4+gAJ/RZv4B3DVmXJGkIXqEqSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWrQgg/rSHIp8EZgT1W9vGu7GnhJt8qBwI+ralOSjcAO4O5u2c1Vdc6oi9bgNp73xZUuQdIKGOQxe5cBnwCumG2oqj+cnU5yIfBIz/r3VNWmURUoSVq8QR6zd1N3RP403cOz3wb8zmjLkiQNY9gx99cAD1bV93rajkjyzSRfTfKa+TZMsjnJVJKp6enpIcuQJPUaNtxPB67qmd8NbKiqo4C/BD6V5Nf7bVhVW6tqsqomJyYmhixDktRryeGe5FnA7wNXz7ZV1RNV9VA3vR24B3jxsEVKkhZnmCP31wF3VdWu2YYkE0n266ZfCBwJ3DtciZKkxVow3JNcBfwP8JIku5Kc3S06jacOyQC8Frg9ybeAzwDnVNXDoyxYkrSwQc6WOX2e9rP6tF0LXDt8WZKkYXiFqiQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwa5cZj0c4PeZfK+LSePuRJJ++KRuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQp0KuQT70WtJCPHKXpAYZ7pLUIMNdkhpkuEtSgwZ5zN6lSfYkuaOn7YNJfpjktu51Us+y9yfZmeTuJL83rsIlSfMb5Mj9MuDEPu0fq6pN3et6gCQvZebZqi/rtvmn2QdmS5KWz4LhXlU3AYM+5PpU4NNV9URV/S+wEzhmiPokSUswzJj7uUlu74ZtDuraDgXu71lnV9f2NEk2J5lKMjU9PT1EGZKkuZYa7hcBLwI2AbuBC7v29Fm3+n1AVW2tqsmqmpyYmFhiGZKkfpYU7lX1YFU9WVU/Ay7mF0Mvu4DDe1Y9DHhguBIlSYu1pHBPsr5n9s3A7Jk024DTkjwnyRHAkcDXhytRkrRYC95bJslVwPHAuiS7gAuA45NsYmbI5T7gnQBVdWeSa4DvAHuBd1XVk+MpXZI0nwXDvapO79N8yT7W/zDw4WGK0jOHz2SVxsMrVCWpQYa7JDXIcJekBhnuktQgn8S0iviEJUmj4pG7JDXIcJekBhnuktQgw12SGmS4S1KDPFtGTVnMGUfe0kAtM9w1Fp7WKa0sh2UkqUGGuyQ1yHCXpAYZ7pLUoAXDPcmlSfYkuaOn7e+S3JXk9iTXJTmwa9+Y5KdJbutenxxn8ZKk/gY5cr8MOHFO2w3Ay6vqFcB3gff3LLunqjZ1r3NGU6YkaTEWDPequgl4eE7bl6tqbzd7M3DYGGqTJC3RKMbc/xj4Us/8EUm+meSrSV4zgs+XJC3SUBcxJfkrYC9wZde0G9hQVQ8leRXwuSQvq6pH+2y7GdgMsGHDhmHKkCTNseQj9yRnAm8E/qiqCqCqnqiqh7rp7cA9wIv7bV9VW6tqsqomJyYmllqGJKmPJYV7khOB9wGnVNXjPe0TSfbrpl8IHAncO4pCJUmDW3BYJslVwPHAuiS7gAuYOTvmOcANSQBu7s6MeS3woSR7gSeBc6rq4b4fLEkamwXDvapO79N8yTzrXgtcO2xRkqTheIWqJDXIW/5qTfAWwtLieOQuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapAXMS0DL8CRtNw8cpekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNGijck1yaZE+SO3ranpfkhiTf694P6tqT5ONJdia5PcnR4ypektTfoEfulwEnzmk7D7ixqo4EbuzmAd7AzIOxjwQ2AxcNX6YkaTEGCvequgmY+6DrU4HLu+nLgTf1tF9RM24GDkyyfhTFSpIGM8yY+yFVtRugez+4az8UuL9nvV1d21Mk2ZxkKsnU9PT0EGVIkuYaxx9U06etntZQtbWqJqtqcmJiYgxlSNIz1zDh/uDscEv3vqdr3wUc3rPeYcADQ3yPJGmRhgn3bcCZ3fSZwOd72t/enTVzLPDI7PCNJGl5DHRXyCRXAccD65LsAi4AtgDXJDkb+AHw1m7164GTgJ3A48A7RlyzJGkBA4V7VZ0+z6IT+qxbwLuGKUpaDoPeivm+LSePuRJp9LxCVZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkho00JOY+knyEuDqnqYXAh8ADgT+FJju2s+vquuXXKG0wnxik9aiJYd7Vd0NbAJIsh/wQ+A6Zp6Z+rGq+shIKpQkLdqohmVOAO6pqu+P6PMkSUMYVbifBlzVM39uktuTXJrkoH4bJNmcZCrJ1PT0dL9VJElLNHS4J3k2cArwr13TRcCLmBmy2Q1c2G+7qtpaVZNVNTkxMTFsGZKkHksec+/xBuDWqnoQYPYdIMnFwBdG8B2r0qB/aJOk5TaKYZnT6RmSSbK+Z9mbgTtG8B2SpEUY6sg9ya8Crwfe2dP8t0k2AQXcN2eZJGkZDBXuVfU48Pw5bWcMVZEkaWheoSpJDTLcJalBhrskNchwl6QGGe6S1KBRXMQkCe8eqdXFI3dJapDhLkkNMtwlqUGOuffhDcEkrXUeuUtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGDX0qZJL7gMeAJ4G9VTWZ5HnA1cBGZp7G9Laq+r9hv0uSNJhRHbn/dlVtqqrJbv484MaqOhK4sZuXJC2TcQ3LnApc3k1fDrxpTN8jSepjFOFewJeTbE+yuWs7pKp2A3TvB4/geyRJAxrF7QeOq6oHkhwM3JDkrkE26n4QbAbYsGHDCMqQJM0a+si9qh7o3vcA1wHHAA8mWQ/Qve/ps93WqpqsqsmJiYlhy5Ak9Rgq3JP8WpLnzk4DvwvcAWwDzuxWOxP4/DDfI0lanGGHZQ4Brksy+1mfqqp/T/IN4JokZwM/AN465PdIkhZhqHCvqnuBV/Zpfwg4YZjPliQtnVeoSlKDDHdJapDhLkkN8jF70jIb9WMc79ty8kg/T23wyF2SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIC9ikta4QS+K8mKnZxaP3CWpQc+oI/dRX/YtSauVR+6S1CDDXZIatORwT3J4kq8k2ZHkziTv7to/mOSHSW7rXieNrlxJ0iCGGXPfC7y3qm7tHpK9PckN3bKPVdVHhi9PkrQUSw73qtoN7O6mH0uyAzh0VIVJGq3FnFDgaZNr30jG3JNsBI4Cbumazk1ye5JLkxw0zzabk0wlmZqenh5FGZKkztDhnuQA4FrgPVX1KHAR8CJgEzNH9hf2266qtlbVZFVNTkxMDFuGJKnHUOGeZH9mgv3KqvosQFU9WFVPVtXPgIuBY4YvU5K0GMOcLRPgEmBHVX20p319z2pvBu5YenmSpKUY5myZ44AzgG8nua1rOx84PckmoID7gHcOVaEkadGGOVvma0D6LLp+6eVIkkbBK1QlqUGGuyQ16Bl1V0hJgxn1HVS9KGr5eeQuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapAXMUkau0EvivJip9HxyF2SGmS4S1KDDHdJalATY+6jvsmRJK11TYS7pDZ4N8rRGVu4JzkR+HtgP+Cfq2rLuL5LkpbLWjnzZyzhnmQ/4B+B1wO7gG8k2VZV3xnH90nSMFoc2h3XkfsxwM6quhcgyaeBUwHDXdKyWcnQXukj/HGF+6HA/T3zu4BX966QZDOwuZv9SZK7x1TLOK0DfrTSRYxAC/2wD6tHC/1Ytj7kb4ba/AXzLRhXuKdPWz1lpmorsHVM378skkxV1eRK1zGsFvphH1aPFvrRQh/GdZ77LuDwnvnDgAfG9F2SpDnGFe7fAI5MckSSZwOnAdvG9F2SpDnGMixTVXuTnAv8BzOnQl5aVXeO47tW2JoeVurRQj/sw+rRQj/WfB9SVQuvJUlaU7y3jCQ1yHCXpAYZ7gtIcmmSPUnumGd5knw8yc4ktyc5erlrHMQA/Tg+ySNJbuteH1juGheS5PAkX0myI8mdSd7dZ51VvT8G7MNa2Be/nOTrSb7V9eOv+6zznCRXd/viliQbl7/S+Q3Yh7OSTPfsiz9ZiVqXpKp87eMFvBY4GrhjnuUnAV9i5tz+Y4FbVrrmJfbjeOALK13nAn1YDxzdTT8X+C7w0rW0Pwbsw1rYFwEO6Kb3B24Bjp2zzp8Dn+ymTwOuXum6l9CHs4BPrHStS3l55L6AqroJeHgfq5wKXFEzbgYOTLJ+eaob3AD9WPWqandV3dpNPwbsYOZq6F6ren8M2IdVr/vv+5Nudv/uNffsjFOBy7vpzwAnJOl3geOKGLAPa5bhPrx+t1pYc/9YO7/V/Yr6pSQvW+li9qX7Ff8oZo62eq2Z/bGPPsAa2BdJ9ktyG7AHuKGq5t0XVbUXeAR4/vJWuW8D9AHgD7ohvs8kObzP8lXJcB/egrdaWCNuBV5QVa8E/gH43ArXM68kBwDXAu+pqkfnLu6zyarbHwv0YU3si6p6sqo2MXMF+jFJXj5nlVW/Lwbow78BG6vqFcB/8ovfRFY9w314Tdxqoaoenf0VtaquB/ZPsm6Fy3qaJPszE4pXVtVn+6yy6vfHQn1YK/tiVlX9GPhv4MQ5i36+L5I8C/gNVunQ4Hx9qKqHquqJbvZi4FXLXNqSGe7D2wa8vTtL41jgkaravdJFLVaS35wdD01yDDP/bzy0slU9VVffJcCOqvroPKut6v0xSB/WyL6YSHJgN/0rwOuAu+astg04s5t+C/Bf1f2VcjUYpA9z/l5zCjN/I1kTfMzeApJcxczZC+uS7AIuYOYPL1TVJ4HrmTlDYyfwOPCOlal03wbox1uAP0uyF/gpcNpq+ofYOQ44A/h2N04KcD6wAdbM/hikD2thX6wHLs/Mg3l+Cbimqr6Q5EPAVFVtY+aH2L8k2cnMEftpK1duX4P04S+SnALsZaYPZ61YtYvk7QckqUEOy0hSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1KD/BymJ1wOmyh3zAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "_=plt.hist(preds_df2[(preds_df2['actual']==1) & (preds_df2['pred']==False)]['epsilon'], bins=30)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Distribution of error in True Negatives is more symmetrical and centered closer to zero\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD4CAYAAAAAczaOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAOJUlEQVR4nO3df6jd9X3H8edr2u6PtmAkV+eSuCsjjLqusyVEh3/Mzi1GLbUtOHRQMydkf0RooWONdSyjTsgoa1lZ58gwqGC1QiuGJkyz0CGD2SWK80ejM7hMr8lMuojtEDbc3vvjfq87Te6Pc3+dc08+zwdczvm+z+ec8z7h5nU++Zzv+SRVhSSpDT8z7AYkSYNj6EtSQwx9SWqIoS9JDTH0Jakh5w67gdmsXr26xsfHh92GJI2Up59++kdVNTbdbSs69MfHxzl06NCw25CkkZLk32a6zeUdSWqIoS9JDTH0Jakhhr4kNcTQl6SGGPqS1BBDX5IaYuhLUkMMfUlqyIr+Rq6khRvfvrfvsUd3Xr+MnWglcaYvSQ0x9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGuIpm5L6Pr3TUztHn6EvjZj5nH8vnc7lHUlqiKEvSQ0x9CWpIYa+JDXED3KlFcIPaDUIzvQlqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQzxlU1Lf3Jht9DnTl6SGGPqS1BBDX5IaYuhLUkMMfUlqiKEvSQ2ZM/STrEvy/SSHk7yY5PNd/fwk+5O80l2u6upJ8o0kR5I8l+TjPY+1pRv/SpIty/eyJEnT6Wem/y7wxar6MHAFsC3JpcB24EBVrQcOdMcA1wLru5+twD0w+SYB7AAuBzYCO6beKCRJgzFn6FfV8ap6prv+E+AwsAa4Abi/G3Y/8Onu+g3AAzXpKeC8JBcB1wD7q+pUVb0F7Ac2L+mrkSTNal5r+knGgY8BPwAurKrjMPnGAFzQDVsDvN5zt4muNlP99OfYmuRQkkMnT56cT3uSpDn0HfpJPgh8B/hCVf14tqHT1GqW+k8XqnZV1Yaq2jA2NtZve5KkPvQV+knex2TgP1hV3+3Kb3bLNnSXJ7r6BLCu5+5rgWOz1CVJA9LP2TsB7gUOV9XXem7aA0ydgbMFeKynfkt3Fs8VwNvd8s/jwKYkq7oPcDd1NUnSgPSzy+aVwOeA55M829W+DOwEHklyG/AacGN32z7gOuAI8A5wK0BVnUpyF3CwG/eVqjq1JK9CktSXOUO/qv6B6dfjAa6eZnwB22Z4rN3A7vk0KElaOu6nL2nJue/+yuU2DJLUEENfkhri8o60zPpd6pAGwZm+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDDH1JaoihL0kNMfQlqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0Jakhhr4kNcTQl6SGGPqS1BBDX5IaYuhLUkMMfUlqiKEvSQ0x9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDzh12A9KoGt++d9gtSPPmTF+SGjJn6CfZneREkhd6an+S5I0kz3Y/1/XcdkeSI0leTnJNT31zVzuSZPvSvxRJ0lz6menfB2yepv71qrqs+9kHkORS4Cbgl7v7/FWSc5KcA3wTuBa4FLi5GytJGqA51/Sr6skk430+3g3Aw1X1X8C/JjkCbOxuO1JVrwIkebgb+8N5dyxJWrDFrOnfnuS5bvlnVVdbA7zeM2aiq81UP0OSrUkOJTl08uTJRbQnSTrdQkP/HuAXgcuA48Cfd/VMM7ZmqZ9ZrNpVVRuqasPY2NgC25MkTWdBp2xW1ZtT15P8DfC97nACWNczdC1wrLs+U12SNCALmuknuajn8DPA1Jk9e4CbkvxskkuA9cA/AQeB9UkuSfJ+Jj/s3bPwtiVJCzHnTD/JQ8BVwOokE8AO4KoklzG5RHMU+H2AqnoxySNMfkD7LrCtqv6ne5zbgceBc4DdVfXikr8aSdKs+jl75+ZpyvfOMv5u4O5p6vuAffPqTpK0pNyGQdLQ9LuVxdGd1y9zJ+1wGwZJaoihL0kNMfQlqSGGviQ1xA9ypdO4T77OZs70Jakhhr4kNcTQl6SGGPqS1BBDX5IaYuhLUkMMfUlqiKEvSQ0x9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDDH1JaoihL0kNMfQlqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0Jakhhr4kNeTcYTcgDcr49r3DbkEaOmf6ktQQQ1+SGjLn8k6S3cAngRNV9ZGudj7wbWAcOAr8dlW9lSTAXwDXAe8Av1tVz3T32QL8Ufewf1pV9y/tS5F0tup3ae7ozuuXuZPR189M/z5g82m17cCBqloPHOiOAa4F1nc/W4F74L03iR3A5cBGYEeSVYttXpI0P3OGflU9CZw6rXwDMDVTvx/4dE/9gZr0FHBekouAa4D9VXWqqt4C9nPmG4kkaZktdE3/wqo6DtBdXtDV1wCv94yb6Goz1c+QZGuSQ0kOnTx5coHtSZKms9Qf5GaaWs1SP7NYtauqNlTVhrGxsSVtTpJat9DQf7NbtqG7PNHVJ4B1PePWAsdmqUuSBmihob8H2NJd3wI81lO/JZOuAN7uln8eBzYlWdV9gLupq0mSBqifUzYfAq4CVieZYPIsnJ3AI0luA14DbuyG72PydM0jTJ6yeStAVZ1KchdwsBv3lao6/cNhSdIymzP0q+rmGW66epqxBWyb4XF2A7vn1Z0kaUn5jVxJaoihL0kNMfQlqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0Jakhhr4kNcTQl6SGzLnLprSSjW/fO+wWpJHiTF+SGmLoS1JDDH1JaoihL0kNMfQlqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0Jakhhr4kNcTQl6SGuJ++pLNGv/+/wtGd1y9zJyuXM31JaoihL0kNMfQlqSGGviQ1xNCXpIYsKvSTHE3yfJJnkxzqaucn2Z/kle5yVVdPkm8kOZLkuSQfX4oXIEnq31LM9D9RVZdV1YbueDtwoKrWAwe6Y4BrgfXdz1bgniV4bknSPCzHefo3AFd11+8H/h74Uld/oKoKeCrJeUkuqqrjy9CDRly/51tLmp/FzvQLeCLJ00m2drULp4K8u7ygq68BXu+570RXkyQNyGJn+ldW1bEkFwD7k7w0y9hMU6szBk2+eWwFuPjiixfZniSp16Jm+lV1rLs8ATwKbATeTHIRQHd5ohs+Aazrufta4Ng0j7mrqjZU1YaxsbHFtCdJOs2CQz/JB5J8aOo6sAl4AdgDbOmGbQEe667vAW7pzuK5Anjb9XxJGqzFLO9cCDyaZOpxvlVVf5vkIPBIktuA14Abu/H7gOuAI8A7wK2LeG5J0gIsOPSr6lXgV6ep/wdw9TT1ArYt9PkkSYvnN3IlqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDVmOrZWlGbllslaC+fweHt15/TJ2MnjO9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDDH1JaoihL0kN8ctZWhJ+6UoaDc70Jakhhr4kNcTQl6SGGPqS1BBDX5IaYuhLUkMMfUlqiOfpa1aef6/W9ft3YFT+sxVn+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0JakhnrLZKE/FlNrkTF+SGuJM/yzjDF4ajlH5EpczfUlqyMBDP8nmJC8nOZJk+6CfX5JaNtDlnSTnAN8EfguYAA4m2VNVPxxkHyuFSzFSe4a9DDToNf2NwJGqehUgycPADcBIhL4hLWnUDTr01wCv9xxPAJf3DkiyFdjaHf5nkpcH0Ndq4EcDeJ6lNqp9w+j2Pqp9g70Pw4L7zp8t6nl/YaYbBh36maZWP3VQtQvYNZh2JiU5VFUbBvmcS2FU+4bR7X1U+wZ7H4aV2PegP8idANb1HK8Fjg24B0lq1qBD/yCwPsklSd4P3ATsGXAPktSsgS7vVNW7SW4HHgfOAXZX1YuD7GEGA11OWkKj2jeMbu+j2jfY+zCsuL5TVXOPkiSdFfxGriQ1xNCXpIYY+j2S/EGSSrJ62L30K8ldSZ5L8mySJ5L8/LB76leSryZ5qev/0STnDbunfiS5McmLSf43yYo6HW8mo7r9SZLdSU4keWHYvcxHknVJvp/kcPe78vlh9zTF0O8kWcfk9hCvDbuXefpqVX20qi4Dvgf88bAbmof9wEeq6qPAvwB3DLmffr0AfBZ4ctiN9KNn+5NrgUuBm5NcOtyu+nYfsHnYTSzAu8AXq+rDwBXAtpXyZ27o/7+vA3/IaV8WW+mq6sc9hx9ghPqvqieq6t3u8Ckmv7ex4lXV4aoaxDfFl8p7259U1X8DU9ufrHhV9SRwath9zFdVHa+qZ7rrPwEOM7kjwdC5nz6Q5FPAG1X1z8l0Xxpe2ZLcDdwCvA18YsjtLNTvAd8edhNnqTm3P9HySTIOfAz4wXA7mdRM6Cf5O+DnprnpTuDLwKbBdtS/2Xqvqseq6k7gziR3ALcDOwba4Czm6r0bcyeT/xx+cJC9zaafvkfInNufaHkk+SDwHeALp/2rfGiaCf2q+s3p6kl+BbgEmJrlrwWeSbKxqv59gC3OaKbep/EtYC8rKPTn6j3JFuCTwNW1gr40Mo8/81Hg9idDkOR9TAb+g1X13WH3M6WZ0J9JVT0PXDB1nOQosKGqRmJHvyTrq+qV7vBTwEvD7Gc+kmwGvgT8elW9M+x+zmLvbX8CvMHk9ie/M9yWzm6ZnEHeCxyuqq8Nu59efpA7+nYmeSHJc0wuUa2YU8P68JfAh4D93Smnfz3shvqR5DNJJoBfA/YmeXzYPc2m+7B8avuTw8AjK2T7kzkleQj4R+CXkkwkuW3YPfXpSuBzwG90v9vPJrlu2E2B2zBIUlOc6UtSQwx9SWqIoS9JDTH0Jakhhr4kNcTQl6SGGPqS1JD/A1MHdS+Ua9mSAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "_=plt.hist(preds_df2[(preds_df2['actual']==0) & (preds_df2['pred']==False)]['epsilon'], bins=30)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### In the cases of False Positive and False Negative: the model made the right inference of \"Likely Conversion\" or \"Unlikely Conversion\", respectively, based solely on the inputs available to it on average.\n",
    "### However it was screwed from the get go due to the large, negative or positive error, respectively, that ended up with the majority of impact in determining the actual conversion status"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Accuracy of Y Prediction\n",
    "### *This is secondary to the main purpose of the model which is to learn attribution percentages, and is mostly unimportant given the simplicity of the DGP in the simulation. However, a model with better performance is more likely to specify the parameters correctly so improving the prediction accuracy can be expected to improve attribution accuracy as well."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import precision_score, recall_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Accuracy: 0.8572\n",
      "Test Precision: 0.2764764104256021\n",
      "Test Recall: 0.2861044725162171\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pred_False</th>\n",
       "      <th>pred_True</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>24878</td>\n",
       "      <td>2193</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2091</td>\n",
       "      <td>838</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   pred_False  pred_True\n",
       "0  24878       2193     \n",
       "1  2091        838      "
      ]
     },
     "execution_count": 166,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "conf = confusion_matrix(preds_df2['actual'], preds_df2['pred'])\n",
    "conf = pd.DataFrame(conf)\n",
    "conf.rename(columns={0:'pred_False', 1:'pred_True'}, inplace=True)\n",
    "print('Test Accuracy:', accuracy_score(preds_df2['actual'], preds_df2['pred']))\n",
    "print('Test Precision:', precision_score(preds_df2['actual'], preds_df2['pred']))\n",
    "print('Test Recall:', recall_score(preds_df2['actual'], preds_df2['pred']))\n",
    "conf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Error Distributions Correlations with Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAS4klEQVR4nO3df4xl5X3f8fenrMG143j5MVC0u/GCsnKCpRo2I4fYVWp70wbWjZeqQcJK6jXdausWR7ZctSW11F+qVPxPSVErqo1xu1QuNiFx2Trkx2bBilprcQYbL2BMWIjDjpayEwPrUBSnuN/+cZ8xl9mZnTsz984uT98v6eqc85zn3POdZw6fOXvOvYdUFZKkvvyFM12AJGn8DHdJ6pDhLkkdMtwlqUOGuyR1aMOZLgDgoosuqq1bt57pMiTpdeWhhx76k6qaWmzdWRHuW7duZWZm5kyXIUmvK0n+eKl1XpaRpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOnRXfUF2LrTf/5hnb97dv+cAZ27cknY5n7pLUIcNdkjpkuEtShwx3SerQsuGe5O1JHh56fTfJJ5JckORgkifb9PzWP0luS3I0yZEk2yf/Y0iShi0b7lX1RFVdWVVXAj8BvAx8EbgZOFRV24BDbRngWmBbe+0Fbp9E4ZKkpa30sswO4Kmq+mNgF7C/te8Hrmvzu4A7a+AwsDHJpWOpVpI0kpWG+w3AXW3+kqp6FqBNL27tm4BjQ9vMtrbXSLI3yUySmbm5uRWWIUk6nZHDPcm5wAeBX1uu6yJtdUpD1b6qmq6q6ampRf8XgJKkVVrJmfu1wNeq6rm2/Nz85ZY2PdHaZ4EtQ9ttBo6vtVBJ0uhW8viBD/HqJRmAA8Bu4JY2vXeo/WNJPg/8JHBy/vKN9Hrj4y30ejVSuCd5E/DXgL831HwLcHeSPcAzwPWt/T5gJ3CUwSdrbhxbtZKkkYwU7lX1MnDhgrbvMPj0zMK+Bdw0luokSaviN1QlqUOGuyR1yHCXpA697v9nHZK0Vj1+Ksozd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjo0Urgn2ZjkniTfSvJ4kp9KckGSg0mebNPzW98kuS3J0SRHkmyf7I8gSVpo1DP3fwf8dlX9GPBO4HHgZuBQVW0DDrVlgGuBbe21F7h9rBVLkpa1bLgn+WHgp4E7AKrqz6vqRWAXsL912w9c1+Z3AXfWwGFgY5JLx165JGlJo5y5Xw7MAf8pydeTfCbJm4FLqupZgDa9uPXfBBwb2n62tUmS1sko4b4B2A7cXlVXAf+bVy/BLCaLtNUpnZK9SWaSzMzNzY1UrCRpNKOE+ywwW1UPtuV7GIT9c/OXW9r0xFD/LUPbbwaOL3zTqtpXVdNVNT01NbXa+iVJi1g23KvqfwHHkry9Ne0AvgkcAHa3tt3AvW3+APDh9qmZq4GT85dvJEnrY8OI/X4J+FySc4GngRsZ/GG4O8ke4Bng+tb3PmAncBR4ufWVJK2jkcK9qh4GphdZtWORvgXctMa6JElr4DdUJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUoZHCPcm3kzyS5OEkM63tgiQHkzzZpue39iS5LcnRJEeSbJ/kDyBJOtVKztzfV1VXVtV0W74ZOFRV24BDbRngWmBbe+0Fbh9XsZKk0azlsswuYH+b3w9cN9R+Zw0cBjYmuXQN+5EkrdCo4V7A7yZ5KMne1nZJVT0L0KYXt/ZNwLGhbWdb22sk2ZtkJsnM3Nzc6qqXJC1qw4j93lNVx5NcDBxM8q3T9M0ibXVKQ9U+YB/A9PT0KeslSas30pl7VR1v0xPAF4F3Ac/NX25p0xOt+yywZWjzzcDxcRUsSVresuGe5M1J3jI/D/x14FHgALC7ddsN3NvmDwAfbp+auRo4OX/5RpK0Pka5LHMJ8MUk8/3/a1X9dpI/AO5Osgd4Bri+9b8P2AkcBV4Gbhx71ZKk01o23KvqaeCdi7R/B9ixSHsBN42lOknSqvgNVUnqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDI4d7knOSfD3Jl9ryZUkeTPJkki8kObe1n9eWj7b1WydTuiRpKSs5c/848PjQ8qeBW6tqG/ACsKe17wFeqKofBW5t/SRJ62ikcE+yGfgA8Jm2HOD9wD2ty37guja/qy3T1u9o/SVJ62TUM/dfAf4x8H/b8oXAi1X1SlueBTa1+U3AMYC2/mTr/xpJ9iaZSTIzNze3yvIlSYtZNtyT/A3gRFU9NNy8SNcaYd2rDVX7qmq6qqanpqZGKlaSNJoNI/R5D/DBJDuBNwI/zOBMfmOSDe3sfDNwvPWfBbYAs0k2AG8Fnh975ZKkJS175l5Vv1xVm6tqK3ADcH9V/QLwAPDzrdtu4N42f6At09bfX1WnnLlLkiZnLZ9z/yfAJ5McZXBN/Y7WfgdwYWv/JHDz2kqUJK3UKJdlfqCqvgx8uc0/DbxrkT5/Blw/htokSavkN1QlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktShZcM9yRuTfDXJN5I8luRftvbLkjyY5MkkX0hybms/ry0fbeu3TvZHkCQtNMqZ+/eA91fVO4ErgWuSXA18Gri1qrYBLwB7Wv89wAtV9aPAra2fJGkdLRvuNfBSW3xDexXwfuCe1r4fuK7N72rLtPU7kmRsFUuSljXSNfck5yR5GDgBHASeAl6sqldal1lgU5vfBBwDaOtPAhcu8p57k8wkmZmbm1vbTyFJeo2Rwr2qvl9VVwKbgXcBP75YtzZd7Cy9Tmmo2ldV01U1PTU1NWq9kqQRrOjTMlX1IvBl4GpgY5INbdVm4HibnwW2ALT1bwWeH0exkqTRjPJpmakkG9v8XwR+BngceAD4+dZtN3Bvmz/Qlmnr76+qU87cJUmTs2H5LlwK7E9yDoM/BndX1ZeSfBP4fJJ/DXwduKP1vwP4L0mOMjhjv2ECdUuSTmPZcK+qI8BVi7Q/zeD6+8L2PwOuH0t1kqRV8RuqktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUoeWDfckW5I8kOTxJI8l+XhrvyDJwSRPtun5rT1JbktyNMmRJNsn/UNIkl5rlDP3V4B/WFU/DlwN3JTkCuBm4FBVbQMOtWWAa4Ft7bUXuH3sVUuSTmvZcK+qZ6vqa23+T4HHgU3ALmB/67YfuK7N7wLurIHDwMYkl469cknSklZ0zT3JVuAq4EHgkqp6FgZ/AICLW7dNwLGhzWZb28L32ptkJsnM3NzcyiuXJC1p5HBP8kPArwOfqKrvnq7rIm11SkPVvqqarqrpqampUcuQJI1gpHBP8gYGwf65qvqN1vzc/OWWNj3R2meBLUObbwaOj6dcSdIoRvm0TIA7gMer6t8OrToA7G7zu4F7h9o/3D41czVwcv7yjSRpfWwYoc97gL8NPJLk4db2T4FbgLuT7AGeAa5v6+4DdgJHgZeBG8dasSRpWcuGe1X9Dxa/jg6wY5H+Bdy0xrokSWvgN1QlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDi0b7kk+m+REkkeH2i5IcjDJk216fmtPktuSHE1yJMn2SRYvSVrcKGfu/xm4ZkHbzcChqtoGHGrLANcC29prL3D7eMqUJK3EsuFeVb8PPL+geRewv83vB64bar+zBg4DG5NcOq5iJUmjWe0190uq6lmANr24tW8Cjg31m21tp0iyN8lMkpm5ublVliFJWsy4b6hmkbZarGNV7auq6aqanpqaGnMZkvT/t9WG+3Pzl1va9ERrnwW2DPXbDBxffXmSpNVYbbgfAHa3+d3AvUPtH26fmrkaODl/+UaStH42LNchyV3Ae4GLkswC/xy4Bbg7yR7gGeD61v0+YCdwFHgZuHECNUuSlrFsuFfVh5ZYtWORvgXctNaiJElr4zdUJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUoYmEe5JrkjyR5GiSmyexD0nS0sYe7knOAf4DcC1wBfChJFeMez+SpKVN4sz9XcDRqnq6qv4c+DywawL7kSQtYcME3nMTcGxoeRb4yYWdkuwF9rbFl5I8scr9XQT8ySq3XZN8+rSrz1hdy7CulTlbjy9wzFbqrKwrn15TXW9basUkwj2LtNUpDVX7gH1r3lkyU1XTa32fcbOulbGulTtba7OulZlUXZO4LDMLbBla3gwcn8B+JElLmES4/wGwLcllSc4FbgAOTGA/kqQljP2yTFW9kuRjwO8A5wCfrarHxr2fIWu+tDMh1rUy1rVyZ2tt1rUyE6krVadcDpckvc75DVVJ6pDhLkkdOqvDfbnHGCQ5L8kX2voHk2wdWvfLrf2JJD+7znV9Msk3kxxJcijJ24bWfT/Jw+011hvNI9T1kSRzQ/v/u0Prdid5sr12r3Ndtw7V9IdJXhxaN8nx+mySE0keXWJ9ktzW6j6SZPvQuomM1wg1/UKr5UiSryR559C6byd5pI3VzLhqWkFt701ycuj39c+G1k3skSQj1PWPhmp6tB1TF7R1ExmzJFuSPJDk8SSPJfn4In0me3xV1Vn5YnAz9ingcuBc4BvAFQv6/APgP7b5G4AvtPkrWv/zgMva+5yzjnW9D3hTm//783W15ZfO4Hh9BPj3i2x7AfB0m57f5s9fr7oW9P8lBjfhJzpe7b1/GtgOPLrE+p3AbzH47sbVwIPrMF7L1fTu+X0xeMTHg0Prvg1cdAbH673Al9Z6DIy7rgV9fw64f9JjBlwKbG/zbwH+cJH/Hid6fJ3NZ+6jPMZgF7C/zd8D7EiS1v75qvpeVf0RcLS937rUVVUPVNXLbfEwg8/6T9paHvvws8DBqnq+ql4ADgLXnKG6PgTcNaZ9n1ZV/T7w/Gm67ALurIHDwMYklzLB8Vqupqr6StsnrN+xNb/v5cZrKRN9JMkK61qX46uqnq2qr7X5PwUeZ/Dt/WETPb7O5nBf7DEGCwfnB32q6hXgJHDhiNtOsq5hexj8dZ73xiQzSQ4nuW5MNa2krr/V/gl4T5L5L5udFePVLl9dBtw/1Dyp8RrFUrVPcrxWYuGxVcDvJnkog8d7nAk/leQbSX4ryTta21kxXknexCAkf32oeeJjlsHl4quABxesmujxNYnHD4zLKI8xWKrPSI9AWKWR3zvJLwLTwF8dav6Rqjqe5HLg/iSPVNVT61TXfwfuqqrvJfkog3/1vH/EbSdZ17wbgHuq6vtDbZMar1GcieNrJEnexyDc/8pQ83vaWF0MHEzyrXZWu16+Brytql5KshP4b8A2zoLxan4O+J9VNXyWP9ExS/JDDP6YfKKqvrtw9SKbjO34OpvP3Ed5jMEP+iTZALyVwT/PJvkIhJHeO8nPAJ8CPlhV35tvr6rjbfo08GUGf9HXpa6q+s5QLb8K/MSo206yriE3sOCfzBMcr1EsVfsZfcRGkr8MfAbYVVXfmW8fGqsTwBcZ36XIkVTVd6vqpTZ/H/CGJBdx9jyS5HTH19jHLMkbGAT756rqNxbpMtnja9w3EsZ4Q2IDgxsJl/HqTZh3LOhzE6+9oXp3m38Hr72h+jTju6E6Sl1XMbiBtG1B+/nAeW3+IuBJxnRjacS6Lh2a/5vA4Xr1Bs4ftfrOb/MXrFddrd/bGdzcynqM19A+trL0DcIP8NobXl+d9HiNUNOPMLiH9O4F7W8G3jI0/xXgmnGO1Qi1/aX53x+DkHymjd1Ix8Ck6mrr50/83rweY9Z+7juBXzlNn4keX2P9xU/gQNrJ4C7zU8CnWtu/YnA2DPBG4Nfawf5V4PKhbT/VtnsCuHad6/o94Dng4fY60NrfDTzSDu5HgD3rXNe/AR5r+38A+LGhbf9OG8ejwI3rWVdb/hfALQu2m/R43QU8C/wfBmdLe4CPAh9t68PgfzzzVNv/9KTHa4SaPgO8MHRszbT2y9s4faP9jj81zrEasbaPDR1fhxn6A7TYMbBedbU+H2HwIYvh7SY2ZgwulxVwZOh3tXM9jy8fPyBJHTqbr7lLklbJcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkd+n+9Zcec79uzkAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "_=plt.hist(preds_df2[(preds_df2['actual']==1) & (preds_df2['pred']==False)]['t2'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### False positives appear to be worse for Google (1) in T1 and Facebook (0) in T2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAS50lEQVR4nO3df6zd9X3f8edrOJAlTWN+XJhluzGoVloiLeBeZW4ydUncreC0MdOKRNQNh3nystEqUaatdJH2S5NG/hkd2sTkhWxmykgobYqX0a6eAVVbZNJLQgyEUAxN4coM3wZwylDTkb33x/nccHx9r++5vudcO58+H9LR+X7f38/3fN/3669f9+vv95zjVBWSpL78ubPdgCRp/Ax3SeqQ4S5JHTLcJalDhrskdWjd2W4A4JJLLqktW7ac7TYk6QfKI4888kdVNbXYsnMi3Lds2cLMzMzZbkOSfqAk+cOllnlZRpI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOnROfEJVks6mLbf8t7O27W/d+qGJvK5n7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdcj3uUun0eP7n/Vng2fuktQhw12SOmS4S1KHDHdJ6tCy4Z7knUkeHXp8J8knklyU5GCSp9vzhW18ktye5GiSI0m2Tf7HkCQNWzbcq+qpqrqqqq4CfgJ4DfgicAtwqKq2AofaPMC1wNb22AvcMYnGJUlLW+lbIXcAz1TVHybZBby/1fcDDwG/DOwC7qqqAg4nWZ9kQ1W9MKaeT+Jb1STpVCu95n4DcHebvmw+sNvzpa2+EXh+aJ3ZVjtJkr1JZpLMzM3NrbANSdLpjBzuSc4HPgz82nJDF6nVKYWqfVU1XVXTU1NTo7YhSRrBSs7crwW+WlUvtvkXk2wAaM/HW30W2Dy03ibg2GoblSSNbiXh/hHeuCQDcADY3aZ3A/cN1W9s75rZDpyY1PV2SdLiRrqhmuQtwF8F/u5Q+VbgniR7gOeA61v9fmAncJTBO2tuGlu3kqSRjBTuVfUacPGC2rcZvHtm4dgCbh5Ld5KkM+InVCWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdWikcE+yPsm9Sb6Z5MkkP5nkoiQHkzzdni9sY5Pk9iRHkxxJsm2yP4IkaaFRz9z/DfDbVfVjwLuBJ4FbgENVtRU41OYBrgW2tsde4I6xdixJWtay4Z7kh4GfAu4EqKo/rapXgF3A/jZsP3Bdm94F3FUDh4H1STaMvXNJ0pJGOXO/ApgD/mOSryX5TJK3ApdV1QsA7fnSNn4j8PzQ+rOtdpIke5PMJJmZm5tb1Q8hSTrZKOG+DtgG3FFVVwP/hzcuwSwmi9TqlELVvqqarqrpqampkZqVJI1mlHCfBWar6uE2fy+DsH9x/nJLez4+NH7z0PqbgGPjaVeSNIplw72q/jfwfJJ3ttIO4BvAAWB3q+0G7mvTB4Ab27tmtgMn5i/fSJLWxroRx/0S8Lkk5wPPAjcx+MVwT5I9wHPA9W3s/cBO4CjwWhsrSVpDI4V7VT0KTC+yaMciYwu4eZV9SZJWwU+oSlKHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nq0EjhnuRbSR5L8miSmVa7KMnBJE+35wtbPUluT3I0yZEk2yb5A0iSTrWSM/cPVNVVVTX/f6neAhyqqq3AoTYPcC2wtT32AneMq1lJ0mhWc1lmF7C/Te8Hrhuq31UDh4H1STasYjuSpBUaNdwL+J0kjyTZ22qXVdULAO350lbfCDw/tO5sq50kyd4kM0lm5ubmzqx7SdKi1o047n1VdSzJpcDBJN88zdgsUqtTClX7gH0A09PTpyyXJJ25kc7cq+pYez4OfBF4D/Di/OWW9ny8DZ8FNg+tvgk4Nq6GJUnLWzbck7w1ydvmp4G/BjwOHAB2t2G7gfva9AHgxvaume3AifnLN5KktTHKZZnLgC8mmR//X6rqt5P8HnBPkj3Ac8D1bfz9wE7gKPAacNPYu5Yknday4V5VzwLvXqT+bWDHIvUCbh5Ld5KkM+InVCWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOjRzuSc5L8rUkX2rzlyd5OMnTSb6Q5PxWv6DNH23Lt0ymdUnSUlZy5v5x4Mmh+U8Dt1XVVuBlYE+r7wFerqofBW5r4yRJa2ikcE+yCfgQ8Jk2H+CDwL1tyH7guja9q83Tlu9o4yVJa2TUM/dfBf4R8P/a/MXAK1X1epufBTa26Y3A8wBt+Yk2/iRJ9iaZSTIzNzd3hu1LkhazbLgn+VngeFU9MlxeZGiNsOyNQtW+qpququmpqamRmpUkjWbdCGPeB3w4yU7gzcAPMziTX59kXTs73wQca+Nngc3AbJJ1wNuBl8beuSRpScueuVfVr1TVpqraAtwAPFBVvwA8CPx8G7YbuK9NH2jztOUPVNUpZ+6SpMlZzfvcfxn4ZJKjDK6p39nqdwIXt/ongVtW16IkaaVGuSzzfVX1EPBQm34WeM8iY/4EuH4MvUmSzpCfUJWkDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1KFlwz3Jm5N8JcnXkzyR5J+3+uVJHk7ydJIvJDm/1S9o80fb8i2T/REkSQuNcub+XeCDVfVu4CrgmiTbgU8Dt1XVVuBlYE8bvwd4uap+FLitjZMkraFlw70GXm2zb2qPAj4I3Nvq+4Hr2vSuNk9bviNJxtaxJGlZI11zT3JekkeB48BB4Bnglap6vQ2ZBTa26Y3A8wBt+Qng4kVec2+SmSQzc3Nzq/spJEknGSncq+p7VXUVsAl4D/Djiw1rz4udpdcphap9VTVdVdNTU1Oj9itJGsGK3i1TVa8ADwHbgfVJ1rVFm4BjbXoW2AzQlr8deGkczUqSRjPKu2Wmkqxv038e+GngSeBB4OfbsN3AfW36QJunLX+gqk45c5ckTc665YewAdif5DwGvwzuqaovJfkG8Pkk/xL4GnBnG38n8J+THGVwxn7DBPqWJJ3GsuFeVUeAqxepP8vg+vvC+p8A14+lO0nSGfETqpLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOjTKf5C9OcmDSZ5M8kSSj7f6RUkOJnm6PV/Y6klye5KjSY4k2TbpH0KSdLJRztxfB/5BVf04sB24OcmVwC3AoaraChxq8wDXAlvbYy9wx9i7liSd1rLhXlUvVNVX2/QfA08CG4FdwP42bD9wXZveBdxVA4eB9Uk2jL1zSdKSVnTNPckW4GrgYeCyqnoBBr8AgEvbsI3A80Orzbbawtfam2Qmyczc3NzKO5ckLWnkcE/yQ8CvA5+oqu+cbugitTqlULWvqqaranpqamrUNiRJIxgp3JO8iUGwf66qfqOVX5y/3NKej7f6LLB5aPVNwLHxtCtJGsUo75YJcCfwZFX966FFB4DdbXo3cN9Q/cb2rpntwIn5yzeSpLWxboQx7wP+FvBYkkdb7R8DtwL3JNkDPAdc35bdD+wEjgKvATeNtWNJ0rKWDfeq+p8sfh0dYMci4wu4eZV9SZJWwU+oSlKHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nq0Cj/QfZnkxxP8vhQ7aIkB5M83Z4vbPUkuT3J0SRHkmybZPOSpMWNcub+n4BrFtRuAQ5V1VbgUJsHuBbY2h57gTvG06YkaSWWDfeq+l3gpQXlXcD+Nr0fuG6oflcNHAbWJ9kwrmYlSaM502vul1XVCwDt+dJW3wg8PzRuttUkSWto3DdUs0itFh2Y7E0yk2Rmbm5uzG1I0p9tZxruL85fbmnPx1t9Ftg8NG4TcGyxF6iqfVU1XVXTU1NTZ9iGJGkxZxruB4DdbXo3cN9Q/cb2rpntwIn5yzeSpLWzbrkBSe4G3g9ckmQW+KfArcA9SfYAzwHXt+H3AzuBo8BrwE0T6FmStIxlw72qPrLEoh2LjC3g5tU2JUlaHT+hKkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHVoIuGe5JokTyU5muSWSWxDkrS0sYd7kvOAfwdcC1wJfCTJlePejiRpaZM4c38PcLSqnq2qPwU+D+yawHYkSUtYN4HX3Ag8PzQ/C/ylhYOS7AX2ttlXkzx1htu7BPijM1x3VfLp0y4+a30tw75W5lw9vsB9tlLnZF/59Kr6esdSCyYR7lmkVqcUqvYB+1a9sWSmqqZX+zrjZl8rY18rd672Zl8rM6m+JnFZZhbYPDS/CTg2ge1IkpYwiXD/PWBrksuTnA/cAByYwHYkSUsY+2WZqno9yS8C/x04D/hsVT0x7u0MWfWlnQmxr5Wxr5U7V3uzr5WZSF+pOuVyuCTpB5yfUJWkDhnuktShczrcl/sagyQXJPlCW/5wki1Dy36l1Z9K8jNr3Ncnk3wjyZEkh5K8Y2jZ95I82h5jvdE8Ql8fTTI3tP2/M7Rsd5Kn22P3Gvd121BPv5/klaFlk9xfn01yPMnjSyxPkttb30eSbBtaNpH9NUJPv9B6OZLky0nePbTsW0kea/tqZlw9raC39yc5MfTn9U+Glk3sK0lG6OsfDvX0eDumLmrLJrLPkmxO8mCSJ5M8keTji4yZ7PFVVefkg8HN2GeAK4Dzga8DVy4Y8/eBf9+mbwC+0KavbOMvAC5vr3PeGvb1AeAtbfrvzffV5l89i/vro8C/XWTdi4Bn2/OFbfrCteprwfhfYnATfqL7q732TwHbgMeXWL4T+C0Gn93YDjy8BvtruZ7eO78tBl/x8fDQsm8Bl5zF/fV+4EurPQbG3deCsT8HPDDpfQZsALa16bcBv7/I38eJHl/n8pn7KF9jsAvY36bvBXYkSat/vqq+W1V/ABxtr7cmfVXVg1X1Wps9zOC9/pO2mq99+BngYFW9VFUvAweBa85SXx8B7h7Ttk+rqn4XeOk0Q3YBd9XAYWB9kg1McH8t11NVfbltE9bu2Jrf9nL7aykT/UqSFfa1JsdXVb1QVV9t038MPMng0/vDJnp8ncvhvtjXGCzcOd8fU1WvAyeAi0dcd5J9DdvD4LfzvDcnmUlyOMl1Y+ppJX39jfZPwHuTzH/Y7JzYX+3y1eXAA0PlSe2vUSzV+yT310osPLYK+J0kj2Tw9R5nw08m+XqS30ryrlY7J/ZXkrcwCMlfHypPfJ9lcLn4auDhBYsmenxN4usHxmWUrzFYasxIX4FwhkZ+7SR/E5gG/spQ+Ueq6liSK4AHkjxWVc+sUV//Fbi7qr6b5GMM/tXzwRHXnWRf824A7q2q7w3VJrW/RnE2jq+RJPkAg3D/y0Pl97V9dSlwMMk321ntWvkq8I6qejXJTuA3ga2cA/ur+Tngf1XV8Fn+RPdZkh9i8MvkE1X1nYWLF1llbMfXuXzmPsrXGHx/TJJ1wNsZ/PNskl+BMNJrJ/lp4FPAh6vqu/P1qjrWnp8FHmLwG31N+qqqbw/18h+Anxh13Un2NeQGFvyTeYL7axRL9X5Wv2IjyV8EPgPsqqpvz9eH9tVx4IuM71LkSKrqO1X1apu+H3hTkks4d76S5HTH19j3WZI3MQj2z1XVbywyZLLH17hvJIzxhsQ6BjcSLueNmzDvWjDmZk6+oXpPm34XJ99QfZbx3VAdpa+rGdxA2rqgfiFwQZu+BHiaMd1YGrGvDUPTfx04XG/cwPmD1t+FbfqiteqrjXsng5tbWYv9NbSNLSx9g/BDnHzD6yuT3l8j9PQjDO4hvXdB/a3A24amvwxcM859NUJvf2H+z49BSD7X9t1Ix8Ck+mrL50/83roW+6z93HcBv3qaMRM9vsb6Bz+BA2kng7vMzwCfarV/weBsGODNwK+1g/0rwBVD636qrfcUcO0a9/U/gBeBR9vjQKu/F3isHdyPAXvWuK9/BTzRtv8g8GND6/7tth+PAjetZV9t/p8Bty5Yb9L7627gBeD/Mjhb2gN8DPhYWx4G//HMM23705PeXyP09Bng5aFja6bVr2j76evtz/hT49xXI/b2i0PH12GGfgEtdgysVV9tzEcZvMlieL2J7TMGl8sKODL0Z7VzLY8vv35Akjp0Ll9zlySdIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdej/A0rKx9Y4NUfuAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "_=plt.hist(preds_df2[(preds_df2['actual']==0) & (preds_df2['pred']==True)]['t1'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAPwUlEQVR4nO3df4xlZX3H8fenrEDBH/xaLV2oC3GjxaYGurGIjVUxUaC6NJUEY+tqt9nYotXStK4lqU3/KSRNsaaNDQWbJTGIRVuoP9pSwDStYe2AyA9XZEEKW6is8kOp8Qf22z/uM3IZZnbO7Nw7Mzx9v5LJnPM8z7nnO8+c/cyZc+aeTVUhSerLj612AZKkyTPcJalDhrskdchwl6QOGe6S1KF1q10AwDHHHFMbN25c7TIk6Rnlpptu+kZVrZ+vb02E+8aNG5mZmVntMiTpGSXJfy7U52UZSeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nq0Jp4h+pybNzx6VXb970XnrVq+5ak/fHMXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdWhQuCf5nSR3JLk9yRVJDk1yQpJdSe5KcmWSg9vYQ9r6nta/cZpfgCTp6RYN9yQbgN8GNlfVzwAHAecCFwEXV9Um4BFgW9tkG/BIVb0IuLiNkyStoKGXZdYBP55kHXAY8CDwWuCq1r8TOLstb2nrtP7Tk2Qy5UqShlg03Kvqv4A/Be5jFOqPATcBj1bVE23YXmBDW94A3N+2faKNP3qyZUuS9mfIZZkjGZ2NnwD8JHA4cMY8Q2t2k/30jb/u9iQzSWb27ds3vGJJ0qKGXJZ5HfC1qtpXVT8APgmcBhzRLtMAHAc80Jb3AscDtP7nAQ/PfdGquqSqNlfV5vXr1y/zy5AkjRsS7vcBpyY5rF07Px34MnAD8OY2ZitwdVu+pq3T+q+vqqeduUuSpmfINfddjG6M3gzc1ra5BHgfcH6SPYyuqV/WNrkMOLq1nw/smELdkqT9WLf4EKiqDwAfmNN8D/DyecZ+Fzhn+aVJkg7UoHCX/r/auOPTq7bvey88a9X2rWc+Hz8gSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1aFC4JzkiyVVJvpJkd5JXJDkqybVJ7mqfj2xjk+RDSfYkuTXJKdP9EiRJcw09c/9z4B+r6iXAy4DdwA7guqraBFzX1gHOADa1j+3AhydasSRpUYuGe5LnAq8CLgOoqu9X1aPAFmBnG7YTOLstbwEur5EbgSOSHDvxyiVJCxpy5n4isA/4myRfTHJpksOBF1TVgwDt8/Pb+A3A/WPb721tkqQVMiTc1wGnAB+uqpOB/+HJSzDzyTxt9bRByfYkM0lm9u3bN6hYSdIwQ8J9L7C3qna19asYhf3XZy+3tM8PjY0/fmz744AH5r5oVV1SVZuravP69esPtH5J0jwWDfeq+m/g/iQvbk2nA18GrgG2tratwNVt+Rrgbe2vZk4FHpu9fCNJWhnrBo57N/DRJAcD9wDvYPSD4eNJtgH3Aee0sZ8BzgT2AN9pYyVJK2hQuFfVLcDmebpOn2dsAectsy5J0jL4DlVJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nq0ND/Zk+SurVxx6dXbd/3XnjWVF7XM3dJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4PDPclBSb6Y5FNt/YQku5LcleTKJAe39kPa+p7Wv3E6pUuSFrKUM/f3ALvH1i8CLq6qTcAjwLbWvg14pKpeBFzcxkmSVtCgcE9yHHAWcGlbD/Ba4Ko2ZCdwdlve0tZp/ae38ZKkFTL0zP2DwO8D/9vWjwYeraon2vpeYENb3gDcD9D6H2vjnyLJ9iQzSWb27dt3gOVLkuazaLgn+SXgoaq6abx5nqE1oO/JhqpLqmpzVW1ev379oGIlScOsGzDmlcCbkpwJHAo8l9GZ/BFJ1rWz8+OAB9r4vcDxwN4k64DnAQ9PvHJJ0oIWPXOvqvdX1XFVtRE4F7i+qt4K3AC8uQ3bClzdlq9p67T+66vqaWfukqTpWc7fub8POD/JHkbX1C9r7ZcBR7f284EdyytRkrRUQy7L/EhVfQ74XFu+B3j5PGO+C5wzgdokSQfId6hKUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOLhnuS45PckGR3kjuSvKe1H5Xk2iR3tc9HtvYk+VCSPUluTXLKtL8ISdJTDTlzfwL43ar6aeBU4LwkJwE7gOuqahNwXVsHOAPY1D62Ax+eeNWSpP1aNNyr6sGqurktfxvYDWwAtgA727CdwNlteQtweY3cCByR5NiJVy5JWtCSrrkn2QicDOwCXlBVD8LoBwDw/DZsA3D/2GZ7W9vc19qeZCbJzL59+5ZeuSRpQYPDPcmzgU8A762qb+1v6Dxt9bSGqkuqanNVbV6/fv3QMiRJAwwK9yTPYhTsH62qT7bmr89ebmmfH2rte4HjxzY/DnhgMuVKkoYY8tcyAS4DdlfVn411XQNsbctbgavH2t/W/mrmVOCx2cs3kqSVsW7AmFcCvwbcluSW1vYHwIXAx5NsA+4Dzml9nwHOBPYA3wHeMdGKJUmLWjTcq+rfmP86OsDp84wv4Lxl1iVJWgbfoSpJHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDk0l3JO8IcmdSfYk2TGNfUiSFjbxcE9yEPCXwBnAScBbkpw06f1IkhY2jTP3lwN7quqeqvo+8DFgyxT2I0lawLopvOYG4P6x9b3Az88dlGQ7sL2tPp7kzgPc3zHANw5w22XJRfvtXrW6FmFdS7NWjy9wzpZqTdaVi5ZV1wsX6phGuGeetnpaQ9UlwCXL3lkyU1Wbl/s6k2ZdS2NdS7dWa7OupZlWXdO4LLMXOH5s/TjggSnsR5K0gGmE+38Am5KckORg4FzgminsR5K0gIlflqmqJ5K8C/gn4CDgI1V1x6T3M2bZl3amxLqWxrqWbq3WZl1LM5W6UvW0y+GSpGc436EqSR0y3CWpQ2s63Bd7jEGSQ5Jc2fp3Jdk41vf+1n5nktevcF3nJ/lykluTXJfkhWN9P0xyS/uY6I3mAXW9Pcm+sf3/xljf1iR3tY+tK1zXxWM1fTXJo2N905yvjyR5KMntC/QnyYda3bcmOWWsbyrzNaCmt7Zabk3y+SQvG+u7N8ltba5mJlXTEmp7dZLHxr5ffzjWN7VHkgyo6/fGarq9HVNHtb6pzFmS45PckGR3kjuSvGeeMdM9vqpqTX4wuhl7N3AicDDwJeCkOWN+C/irtnwucGVbPqmNPwQ4ob3OQStY12uAw9ryb87W1dYfX8X5ejvwF/NsexRwT/t8ZFs+cqXqmjP+3Yxuwk91vtprvwo4Bbh9gf4zgc8yeu/GqcCuFZivxWo6bXZfjB7xsWus717gmFWcr1cDn1ruMTDpuuaMfSNw/bTnDDgWOKUtPwf46jz/Hqd6fK3lM/chjzHYAuxsy1cBpydJa/9YVX2vqr4G7GmvtyJ1VdUNVfWdtnojo7/1n7blPPbh9cC1VfVwVT0CXAu8YZXqegtwxYT2vV9V9a/Aw/sZsgW4vEZuBI5IcixTnK/Faqqqz7d9wsodW7P7Xmy+FjLVR5Issa4VOb6q6sGqurktfxvYzejd++Omenyt5XCf7zEGcyfnR2Oq6gngMeDogdtOs65x2xj9dJ51aJKZJDcmOXtCNS2lrl9pvwJelWT2zWZrYr7a5asTgOvHmqc1X0MsVPs052sp5h5bBfxzkpsyerzHanhFki8l+WySl7a2NTFfSQ5jFJKfGGue+pxldLn4ZGDXnK6pHl/TePzApAx5jMFCYwY9AuEADX7tJL8KbAZ+caz5p6rqgSQnAtcnua2q7l6huv4BuKKqvpfknYx+63ntwG2nWdesc4GrquqHY23Tmq8hVuP4GiTJaxiF+y+MNb+yzdXzgWuTfKWd1a6Um4EXVtXjSc4E/h7YxBqYr+aNwL9X1fhZ/lTnLMmzGf0weW9VfWtu9zybTOz4Wstn7kMeY/CjMUnWAc9j9OvZNB+BMOi1k7wOuAB4U1V9b7a9qh5on+8BPsfoJ/qK1FVV3xyr5a+Bnxu67TTrGnMuc35lnuJ8DbFQ7av6iI0kPwtcCmypqm/Oto/N1UPA3zG5S5GDVNW3qurxtvwZ4FlJjmHtPJJkf8fXxOcsybMYBftHq+qT8wyZ7vE16RsJE7whsY7RjYQTePImzEvnjDmPp95Q/XhbfilPvaF6D5O7oTqkrpMZ3UDaNKf9SOCQtnwMcBcTurE0sK5jx5Z/GbixnryB87VW35Ft+aiVqquNezGjm1tZifka28dGFr5BeBZPveH1hWnP14CaforRPaTT5rQfDjxnbPnzwBsmOVcDavuJ2e8fo5C8r83doGNgWnW1/tkTv8NXYs7a13058MH9jJnq8TXRb/wUDqQzGd1lvhu4oLX9MaOzYYBDgb9tB/sXgBPHtr2gbXcncMYK1/UvwNeBW9rHNa39NOC2dnDfBmxb4br+BLij7f8G4CVj2/56m8c9wDtWsq62/kfAhXO2m/Z8XQE8CPyA0dnSNuCdwDtbfxj9xzN3t/1vnvZ8DajpUuCRsWNrprWf2ObpS+17fMEk52pgbe8aO75uZOwH0HzHwErV1ca8ndEfWYxvN7U5Y3S5rIBbx75XZ67k8eXjBySpQ2v5mrsk6QAZ7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalD/wfZrY1Ule1M7gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "_=plt.hist(preds_df2[(preds_df2['actual']==0) & (preds_df2['pred']==True)]['t2'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### False Negatives appear to be worse for Facebook (0) in T2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAS7UlEQVR4nO3df4xl5X3f8fenrMG143r5MVC0u/GCsnKCqxrIyKV2ldombQAnXqqGCista7LV1i2JbLlqS2qpv1Sp+J/ioFZEW3C7VA6GkLhsHZKGLlhRay3OgDE/TAgLIbBayk4MrENRnOJ++8d9JlxmZ3bO7Nw7uzx9v6Sr+5znec493zl7+MzhnHvvpKqQJPXlz5zoAiRJk2e4S1KHDHdJ6pDhLkkdMtwlqUMbTnQBAGeddVZt3br1RJchSW8pDz744B9W1cxSYydFuG/dupW5ubkTXYYkvaUk+YPlxrwsI0kdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTopPqG6Fluv//UTtu1nb/jYCdu2JB2LZ+6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktSht/wnVCVprXr8pLtn7pLUoRXDPcl7kzw89vhuks8kOSPJvUmeas+nt/lJclOSA0keSXLx9H8MSdK4FcO9qp6sqgur6kLgR4HXgK8A1wP7qmobsK8tA1wObGuPXcDN0yhckrS81V6WuRR4uqr+ANgO7Gn9e4ArW3s7cFuN7Ac2Jjl3ItVKkgZZbbhfDdze2udU1QsA7fns1r8JeH5snYOtT5K0TgaHe5JTgY8Dv7LS1CX6aonX25VkLsnc/Pz80DIkSQOs5sz9cuChqnqxLb+4cLmlPR9u/QeBLWPrbQYOLX6xqtpdVbNVNTszM7P6yiVJy1pNuH+CNy7JAOwFdrT2DuDusf5r2rtmLgGOLFy+kSStj0EfYkryDuCvAX9vrPsG4M4kO4HngKta/z3AFcABRu+suXZi1UqSBhkU7lX1GnDmor7vMHr3zOK5BVw3keokScfFT6hKUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHRr6B7I3ArcAfwEo4GeBJ4E7gK3As8DfqqqXkwT4RUZ/JPs14JNV9dDEK5fWwdbrf/2EbfvZGz52wratt76hZ+6/CPxmVf0w8H7gCeB6YF9VbQP2tWWAy4Ft7bELuHmiFUuSVrRiuCf5c8CPAbcCVNWfVNUrwHZgT5u2B7iytbcDt9XIfmBjknMnXrkkaVlDztzPB+aB/5jkm0luSfJO4JyqegGgPZ/d5m8Cnh9b/2Dre5Mku5LMJZmbn59f0w8hSXqzIeG+AbgYuLmqLgL+N29cgllKluirozqqdlfVbFXNzszMDCpWkjTMkHA/CBysqgfa8l2Mwv7Fhcst7fnw2PwtY+tvBg5NplxJ0hArhntV/S/g+STvbV2XAt8G9gI7Wt8O4O7W3gtck5FLgCMLl28kSetj0FshgZ8HvpTkVOAZ4FpGvxjuTLITeA64qs29h9HbIA8weivktROtWJK0okHhXlUPA7NLDF26xNwCrltjXZKkNfATqpLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOjQo3JM8m+TRJA8nmWt9ZyS5N8lT7fn01p8kNyU5kOSRJBdP8weQJB1tNWfuH6mqC6tq4W+pXg/sq6ptwL62DHA5sK09dgE3T6pYSdIwa7kssx3Y09p7gCvH+m+rkf3AxiTnrmE7kqRVGhruBfxWkgeT7Gp951TVCwDt+ezWvwl4fmzdg63vTZLsSjKXZG5+fv74qpckLWnDwHkfqqpDSc4G7k3yu8eYmyX66qiOqt3AboDZ2dmjxiVJx2/QmXtVHWrPh4GvAB8AXly43NKeD7fpB4EtY6tvBg5NqmBJ0spWDPck70zyroU28NeBx4C9wI42bQdwd2vvBa5p75q5BDiycPlGkrQ+hlyWOQf4SpKF+b9cVb+Z5HeAO5PsBJ4Drmrz7wGuAA4ArwHXTrxqSdIxrRjuVfUM8P4l+r8DXLpEfwHXTaQ6SdJx8ROqktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6NDjck5yS5JtJvtqWz0vyQJKnktyR5NTWf1pbPtDGt06ndEnSclZz5v5p4Imx5c8DN1bVNuBlYGfr3wm8XFU/BNzY5kmS1tGgcE+yGfgYcEtbDvBR4K42ZQ9wZWtvb8u08UvbfEnSOhl65v4F4B8D/7ctnwm8UlWvt+WDwKbW3gQ8D9DGj7T5kqR1smK4J/lJ4HBVPTjevcTUGjA2/rq7kswlmZufnx9UrCRpmCFn7h8CPp7kWeDLjC7HfAHYmGRDm7MZONTaB4EtAG383cBLi1+0qnZX1WxVzc7MzKzph5AkvdmK4V5Vv1BVm6tqK3A1cF9V/QxwP/DTbdoO4O7W3tuWaeP3VdVRZ+6SpOlZy/vc/wnw2SQHGF1Tv7X13wqc2fo/C1y/thIlSau1YeUpb6iqrwFfa+1ngA8sMeePgasmUJsk6Tj5CVVJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQyuGe5K3J/lGkm8leTzJv2z95yV5IMlTSe5IcmrrP60tH2jjW6f7I0iSFhty5v494KNV9X7gQuCyJJcAnwdurKptwMvAzjZ/J/ByVf0QcGObJ0laRyuGe4282hbf1h4FfBS4q/XvAa5s7e1tmTZ+aZJMrGJJ0ooGXXNPckqSh4HDwL3A08ArVfV6m3IQ2NTam4DnAdr4EeDMJV5zV5K5JHPz8/Nr+ykkSW8yKNyr6vtVdSGwGfgA8CNLTWvPS52l11EdVburaraqZmdmZobWK0kaYFXvlqmqV4CvAZcAG5NsaEObgUOtfRDYAtDG3w28NIliJUnDDHm3zEySja39Z4EfB54A7gd+uk3bAdzd2nvbMm38vqo66sxdkjQ9G1aewrnAniSnMPplcGdVfTXJt4EvJ/nXwDeBW9v8W4H/nOQAozP2q6dQtyTpGFYM96p6BLhoif5nGF1/X9z/x8BVE6lOknRc/ISqJHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QODfkD2VuS3J/kiSSPJ/l06z8jyb1JnmrPp7f+JLkpyYEkjyS5eNo/hCTpzYacub8O/MOq+hHgEuC6JBcA1wP7qmobsK8tA1wObGuPXcDNE69aknRMK4Z7Vb1QVQ+19h8BTwCbgO3AnjZtD3Bla28HbquR/cDGJOdOvHJJ0rJWdc09yVbgIuAB4JyqegFGvwCAs9u0TcDzY6sdbH2LX2tXkrkkc/Pz86uvXJK0rMHhnuQHgF8FPlNV3z3W1CX66qiOqt1VNVtVszMzM0PLkCQNMCjck7yNUbB/qap+rXW/uHC5pT0fbv0HgS1jq28GDk2mXEnSEEPeLRPgVuCJqvq3Y0N7gR2tvQO4e6z/mvaumUuAIwuXbyRJ62PDgDkfAv4O8GiSh1vfPwVuAO5MshN4Driqjd0DXAEcAF4Drp1oxZKkFa0Y7lX1P1j6OjrApUvML+C6NdYlSVoDP6EqSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDQ/5A9heTHE7y2FjfGUnuTfJUez699SfJTUkOJHkkycXTLF6StLQhZ+7/CbhsUd/1wL6q2gbsa8sAlwPb2mMXcPNkypQkrcaK4V5Vvw28tKh7O7CntfcAV47131Yj+4GNSc6dVLGSpGGO95r7OVX1AkB7Prv1bwKeH5t3sPUdJcmuJHNJ5ubn54+zDEnSUiZ9QzVL9NVSE6tqd1XNVtXszMzMhMuQpP+/HW+4v7hwuaU9H279B4EtY/M2A4eOvzxJ0vE43nDfC+xo7R3A3WP917R3zVwCHFm4fCNJWj8bVpqQ5Hbgw8BZSQ4C/xy4AbgzyU7gOeCqNv0e4ArgAPAacO0UapYkrWDFcK+qTywzdOkScwu4bq1FSZLWxk+oSlKHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR2aSrgnuSzJk0kOJLl+GtuQJC1v4uGe5BTg3wOXAxcAn0hywaS3I0la3jTO3D8AHKiqZ6rqT4AvA9unsB1J0jI2TOE1NwHPjy0fBP7S4klJdgG72uKrSZ48zu2dBfzhca67Jvn8MYdPWF0rsK7VOVmPL3CfrdZJWVc+v6a63rPcwDTCPUv01VEdVbuB3WveWDJXVbNrfZ1Js67Vsa7VO1lrs67VmVZd07gscxDYMra8GTg0he1IkpYxjXD/HWBbkvOSnApcDeydwnYkScuY+GWZqno9yc8B/w04BfhiVT0+6e2MWfOlnSmxrtWxrtU7WWuzrtWZSl2pOupyuCTpLc5PqEpShwx3SerQSR3uK32NQZLTktzRxh9IsnVs7Bda/5NJfmKd6/pskm8neSTJviTvGRv7fpKH22OiN5oH1PXJJPNj2/+7Y2M7kjzVHjvWua4bx2r6vSSvjI1Nc399McnhJI8tM54kN7W6H0ly8djYVPbXgJp+ptXySJKvJ3n/2NizSR5t+2puUjWtorYPJzky9u/1z8bGpvaVJAPq+kdjNT3Wjqkz2thU9lmSLUnuT/JEkseTfHqJOdM9vqrqpHwwuhn7NHA+cCrwLeCCRXP+AfBLrX01cEdrX9Dmnwac117nlHWs6yPAO1r77y/U1ZZfPYH765PAv1ti3TOAZ9rz6a19+nrVtWj+zzO6CT/V/dVe+8eAi4HHlhm/AvgNRp/duAR4YB3210o1fXBhW4y+4uOBsbFngbNO4P76MPDVtR4Dk65r0dyfAu6b9j4DzgUubu13Ab+3xH+PUz2+TuYz9yFfY7Ad2NPadwGXJknr/3JVfa+qfh840F5vXeqqqvur6rW2uJ/Re/2nbS1f+/ATwL1V9VJVvQzcC1x2gur6BHD7hLZ9TFX128BLx5iyHbitRvYDG5OcyxT310o1VdXX2zZh/Y6thW2vtL+WM9WvJFllXetyfFXVC1X1UGv/EfAEo0/vj5vq8XUyh/tSX2OweOf86Zyqeh04Apw5cN1p1jVuJ6PfzgvenmQuyf4kV06optXU9Tfb/wLelWThw2Ynxf5ql6/OA+4b657W/hpiudqnub9WY/GxVcBvJXkwo6/3OBH+cpJvJfmNJO9rfSfF/kryDkYh+atj3VPfZxldLr4IeGDR0FSPr2l8/cCkDPkag+XmDPoKhOM0+LWT/G1gFvirY90/WFWHkpwP3Jfk0ap6ep3q+q/A7VX1vSSfYvR/PR8duO4061pwNXBXVX1/rG9a+2uIE3F8DZLkI4zC/a+MdX+o7auzgXuT/G47q10vDwHvqapXk1wB/BdgGyfB/mp+CvifVTV+lj/VfZbkBxj9MvlMVX138fASq0zs+DqZz9yHfI3Bn85JsgF4N6P/PZvmVyAMeu0kPw58Dvh4VX1vob+qDrXnZ4CvMfqNvi51VdV3xmr5D8CPDl13mnWNuZpF/8s8xf01xHK1n9Cv2EjyF4FbgO1V9Z2F/rF9dRj4CpO7FDlIVX23ql5t7XuAtyU5i5PnK0mOdXxNfJ8leRujYP9SVf3aElOme3xN+kbCBG9IbGB0I+E83rgJ875Fc67jzTdU72zt9/HmG6rPMLkbqkPquojRDaRti/pPB05r7bOAp5jQjaWBdZ071v4bwP564wbO77f6Tm/tM9arrjbvvYxubmU99tfYNray/A3Cj/HmG17fmPb+GlDTDzK6h/TBRf3vBN411v46cNkk99WA2v78wr8fo5B8ru27QcfAtOpq4wsnfu9cj33Wfu7bgC8cY85Uj6+J/sNP4UC6gtFd5qeBz7W+f8XobBjg7cCvtIP9G8D5Y+t+rq33JHD5Otf134EXgYfbY2/r/yDwaDu4HwV2rnNd/wZ4vG3/fuCHx9b92bYfDwDXrmddbflfADcsWm/a++t24AXg/zA6W9oJfAr4VBsPoz8883Tb/uy099eAmm4BXh47tuZa//ltP32r/Rt/bpL7amBtPzd2fO1n7BfQUsfAetXV5nyS0Zssxteb2j5jdLmsgEfG/q2uWM/jy68fkKQOnczX3CVJx8lwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR36fxmGyjbm6lrfAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "_=plt.hist(preds_df2[(preds_df2['actual']==1) & (preds_df2['pred']==False)]['t1'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAS4klEQVR4nO3df4xl5X3f8fenrMG143j5MVC0u/GCsnKCpRo2I4fYVWp70wbWjZeqQcJK6jXdausWR7ZctSW11F+qVPxPSVErqo1xu1QuNiFx2Trkx2bBilprcQYbL2BMWIjDjpayEwPrUBSnuN/+cZ8xl9mZnTsz984uT98v6eqc85zn3POdZw6fOXvOvYdUFZKkvvyFM12AJGn8DHdJ6pDhLkkdMtwlqUOGuyR1aMOZLgDgoosuqq1bt57pMiTpdeWhhx76k6qaWmzdWRHuW7duZWZm5kyXIUmvK0n+eKl1XpaRpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOnRXfUF2LrTf/5hnb97dv+cAZ27cknY5n7pLUIcNdkjpkuEtShwx3SerQsuGe5O1JHh56fTfJJ5JckORgkifb9PzWP0luS3I0yZEk2yf/Y0iShi0b7lX1RFVdWVVXAj8BvAx8EbgZOFRV24BDbRngWmBbe+0Fbp9E4ZKkpa30sswO4Kmq+mNgF7C/te8Hrmvzu4A7a+AwsDHJpWOpVpI0kpWG+w3AXW3+kqp6FqBNL27tm4BjQ9vMtrbXSLI3yUySmbm5uRWWIUk6nZHDPcm5wAeBX1uu6yJtdUpD1b6qmq6q6ampRf8XgJKkVVrJmfu1wNeq6rm2/Nz85ZY2PdHaZ4EtQ9ttBo6vtVBJ0uhW8viBD/HqJRmAA8Bu4JY2vXeo/WNJPg/8JHBy/vKN9Hrj4y30ejVSuCd5E/DXgL831HwLcHeSPcAzwPWt/T5gJ3CUwSdrbhxbtZKkkYwU7lX1MnDhgrbvMPj0zMK+Bdw0luokSaviN1QlqUOGuyR1yHCXpA697v9nHZK0Vj1+Ksozd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjo0Urgn2ZjkniTfSvJ4kp9KckGSg0mebNPzW98kuS3J0SRHkmyf7I8gSVpo1DP3fwf8dlX9GPBO4HHgZuBQVW0DDrVlgGuBbe21F7h9rBVLkpa1bLgn+WHgp4E7AKrqz6vqRWAXsL912w9c1+Z3AXfWwGFgY5JLx165JGlJo5y5Xw7MAf8pydeTfCbJm4FLqupZgDa9uPXfBBwb2n62tUmS1sko4b4B2A7cXlVXAf+bVy/BLCaLtNUpnZK9SWaSzMzNzY1UrCRpNKOE+ywwW1UPtuV7GIT9c/OXW9r0xFD/LUPbbwaOL3zTqtpXVdNVNT01NbXa+iVJi1g23KvqfwHHkry9Ne0AvgkcAHa3tt3AvW3+APDh9qmZq4GT85dvJEnrY8OI/X4J+FySc4GngRsZ/GG4O8ke4Bng+tb3PmAncBR4ufWVJK2jkcK9qh4GphdZtWORvgXctMa6JElr4DdUJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUoZHCPcm3kzyS5OEkM63tgiQHkzzZpue39iS5LcnRJEeSbJ/kDyBJOtVKztzfV1VXVtV0W74ZOFRV24BDbRngWmBbe+0Fbh9XsZKk0azlsswuYH+b3w9cN9R+Zw0cBjYmuXQN+5EkrdCo4V7A7yZ5KMne1nZJVT0L0KYXt/ZNwLGhbWdb22sk2ZtkJsnM3Nzc6qqXJC1qw4j93lNVx5NcDBxM8q3T9M0ibXVKQ9U+YB/A9PT0KeslSas30pl7VR1v0xPAF4F3Ac/NX25p0xOt+yywZWjzzcDxcRUsSVresuGe5M1J3jI/D/x14FHgALC7ddsN3NvmDwAfbp+auRo4OX/5RpK0Pka5LHMJ8MUk8/3/a1X9dpI/AO5Osgd4Bri+9b8P2AkcBV4Gbhx71ZKk01o23KvqaeCdi7R/B9ixSHsBN42lOknSqvgNVUnqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDI4d7knOSfD3Jl9ryZUkeTPJkki8kObe1n9eWj7b1WydTuiRpKSs5c/848PjQ8qeBW6tqG/ACsKe17wFeqKofBW5t/SRJ62ikcE+yGfgA8Jm2HOD9wD2ty37guja/qy3T1u9o/SVJ62TUM/dfAf4x8H/b8oXAi1X1SlueBTa1+U3AMYC2/mTr/xpJ9iaZSTIzNze3yvIlSYtZNtyT/A3gRFU9NNy8SNcaYd2rDVX7qmq6qqanpqZGKlaSNJoNI/R5D/DBJDuBNwI/zOBMfmOSDe3sfDNwvPWfBbYAs0k2AG8Fnh975ZKkJS175l5Vv1xVm6tqK3ADcH9V/QLwAPDzrdtu4N42f6At09bfX1WnnLlLkiZnLZ9z/yfAJ5McZXBN/Y7WfgdwYWv/JHDz2kqUJK3UKJdlfqCqvgx8uc0/DbxrkT5/Blw/htokSavkN1QlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktShZcM9yRuTfDXJN5I8luRftvbLkjyY5MkkX0hybms/ry0fbeu3TvZHkCQtNMqZ+/eA91fVO4ErgWuSXA18Gri1qrYBLwB7Wv89wAtV9aPAra2fJGkdLRvuNfBSW3xDexXwfuCe1r4fuK7N72rLtPU7kmRsFUuSljXSNfck5yR5GDgBHASeAl6sqldal1lgU5vfBBwDaOtPAhcu8p57k8wkmZmbm1vbTyFJeo2Rwr2qvl9VVwKbgXcBP75YtzZd7Cy9Tmmo2ldV01U1PTU1NWq9kqQRrOjTMlX1IvBl4GpgY5INbdVm4HibnwW2ALT1bwWeH0exkqTRjPJpmakkG9v8XwR+BngceAD4+dZtN3Bvmz/Qlmnr76+qU87cJUmTs2H5LlwK7E9yDoM/BndX1ZeSfBP4fJJ/DXwduKP1vwP4L0mOMjhjv2ECdUuSTmPZcK+qI8BVi7Q/zeD6+8L2PwOuH0t1kqRV8RuqktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUoeWDfckW5I8kOTxJI8l+XhrvyDJwSRPtun5rT1JbktyNMmRJNsn/UNIkl5rlDP3V4B/WFU/DlwN3JTkCuBm4FBVbQMOtWWAa4Ft7bUXuH3sVUuSTmvZcK+qZ6vqa23+T4HHgU3ALmB/67YfuK7N7wLurIHDwMYkl469cknSklZ0zT3JVuAq4EHgkqp6FgZ/AICLW7dNwLGhzWZb28L32ptkJsnM3NzcyiuXJC1p5HBP8kPArwOfqKrvnq7rIm11SkPVvqqarqrpqampUcuQJI1gpHBP8gYGwf65qvqN1vzc/OWWNj3R2meBLUObbwaOj6dcSdIoRvm0TIA7gMer6t8OrToA7G7zu4F7h9o/3D41czVwcv7yjSRpfWwYoc97gL8NPJLk4db2T4FbgLuT7AGeAa5v6+4DdgJHgZeBG8dasSRpWcuGe1X9Dxa/jg6wY5H+Bdy0xrokSWvgN1QlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDi0b7kk+m+REkkeH2i5IcjDJk216fmtPktuSHE1yJMn2SRYvSVrcKGfu/xm4ZkHbzcChqtoGHGrLANcC29prL3D7eMqUJK3EsuFeVb8PPL+geRewv83vB64bar+zBg4DG5NcOq5iJUmjWe0190uq6lmANr24tW8Cjg31m21tp0iyN8lMkpm5ublVliFJWsy4b6hmkbZarGNV7auq6aqanpqaGnMZkvT/t9WG+3Pzl1va9ERrnwW2DPXbDBxffXmSpNVYbbgfAHa3+d3AvUPtH26fmrkaODl/+UaStH42LNchyV3Ae4GLkswC/xy4Bbg7yR7gGeD61v0+YCdwFHgZuHECNUuSlrFsuFfVh5ZYtWORvgXctNaiJElr4zdUJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUoYmEe5JrkjyR5GiSmyexD0nS0sYe7knOAf4DcC1wBfChJFeMez+SpKVN4sz9XcDRqnq6qv4c+DywawL7kSQtYcME3nMTcGxoeRb4yYWdkuwF9rbFl5I8scr9XQT8ySq3XZN8+rSrz1hdy7CulTlbjy9wzFbqrKwrn15TXW9basUkwj2LtNUpDVX7gH1r3lkyU1XTa32fcbOulbGulTtba7OulZlUXZO4LDMLbBla3gwcn8B+JElLmES4/wGwLcllSc4FbgAOTGA/kqQljP2yTFW9kuRjwO8A5wCfrarHxr2fIWu+tDMh1rUy1rVyZ2tt1rUyE6krVadcDpckvc75DVVJ6pDhLkkdOqvDfbnHGCQ5L8kX2voHk2wdWvfLrf2JJD+7znV9Msk3kxxJcijJ24bWfT/Jw+011hvNI9T1kSRzQ/v/u0Prdid5sr12r3Ndtw7V9IdJXhxaN8nx+mySE0keXWJ9ktzW6j6SZPvQuomM1wg1/UKr5UiSryR559C6byd5pI3VzLhqWkFt701ycuj39c+G1k3skSQj1PWPhmp6tB1TF7R1ExmzJFuSPJDk8SSPJfn4In0me3xV1Vn5YnAz9ingcuBc4BvAFQv6/APgP7b5G4AvtPkrWv/zgMva+5yzjnW9D3hTm//783W15ZfO4Hh9BPj3i2x7AfB0m57f5s9fr7oW9P8lBjfhJzpe7b1/GtgOPLrE+p3AbzH47sbVwIPrMF7L1fTu+X0xeMTHg0Prvg1cdAbH673Al9Z6DIy7rgV9fw64f9JjBlwKbG/zbwH+cJH/Hid6fJ3NZ+6jPMZgF7C/zd8D7EiS1v75qvpeVf0RcLS937rUVVUPVNXLbfEwg8/6T9paHvvws8DBqnq+ql4ADgLXnKG6PgTcNaZ9n1ZV/T7w/Gm67ALurIHDwMYklzLB8Vqupqr6StsnrN+xNb/v5cZrKRN9JMkK61qX46uqnq2qr7X5PwUeZ/Dt/WETPb7O5nBf7DEGCwfnB32q6hXgJHDhiNtOsq5hexj8dZ73xiQzSQ4nuW5MNa2krr/V/gl4T5L5L5udFePVLl9dBtw/1Dyp8RrFUrVPcrxWYuGxVcDvJnkog8d7nAk/leQbSX4ryTta21kxXknexCAkf32oeeJjlsHl4quABxesmujxNYnHD4zLKI8xWKrPSI9AWKWR3zvJLwLTwF8dav6Rqjqe5HLg/iSPVNVT61TXfwfuqqrvJfkog3/1vH/EbSdZ17wbgHuq6vtDbZMar1GcieNrJEnexyDc/8pQ83vaWF0MHEzyrXZWu16+Brytql5KshP4b8A2zoLxan4O+J9VNXyWP9ExS/JDDP6YfKKqvrtw9SKbjO34OpvP3Ed5jMEP+iTZALyVwT/PJvkIhJHeO8nPAJ8CPlhV35tvr6rjbfo08GUGf9HXpa6q+s5QLb8K/MSo206yriE3sOCfzBMcr1EsVfsZfcRGkr8MfAbYVVXfmW8fGqsTwBcZ36XIkVTVd6vqpTZ/H/CGJBdx9jyS5HTH19jHLMkbGAT756rqNxbpMtnja9w3EsZ4Q2IDgxsJl/HqTZh3LOhzE6+9oXp3m38Hr72h+jTju6E6Sl1XMbiBtG1B+/nAeW3+IuBJxnRjacS6Lh2a/5vA4Xr1Bs4ftfrOb/MXrFddrd/bGdzcynqM19A+trL0DcIP8NobXl+d9HiNUNOPMLiH9O4F7W8G3jI0/xXgmnGO1Qi1/aX53x+DkHymjd1Ix8Ck6mrr50/83rweY9Z+7juBXzlNn4keX2P9xU/gQNrJ4C7zU8CnWtu/YnA2DPBG4Nfawf5V4PKhbT/VtnsCuHad6/o94Dng4fY60NrfDTzSDu5HgD3rXNe/AR5r+38A+LGhbf9OG8ejwI3rWVdb/hfALQu2m/R43QU8C/wfBmdLe4CPAh9t68PgfzzzVNv/9KTHa4SaPgO8MHRszbT2y9s4faP9jj81zrEasbaPDR1fhxn6A7TYMbBedbU+H2HwIYvh7SY2ZgwulxVwZOh3tXM9jy8fPyBJHTqbr7lLklbJcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkd+n+9Zcec79uzkAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "_=plt.hist(preds_df2[(preds_df2['actual']==1) & (preds_df2['pred']==False)]['t2'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Elapsed time (gap1) distribution for False Negative"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAPlklEQVR4nO3db6yedX3H8fdnVPyD04IcCGubHYyNf7IEISesjsQ46hb+GMsDSTCbNKRJnzCH08RVn5gle1CTRZRkIWlALRtTCWpohDhJwZg9gHkQhmA1dIzRs1Z6HH/8Q5xjfvfg/KqH9i7nbs999z799f1KTq7r+l6/+1zfK20+59ffue+rqSokSX35nUk3IEkaPcNdkjpkuEtShwx3SeqQ4S5JHVo16QYAzj777Jqenp50G5J0UnnooYd+UlVTg86tiHCfnp5mdnZ20m1I0kklyX8e7ZzLMpLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1KEV8QnVk9X0trsnct2ntl85ketKOnk4c5ekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdWiocE+yOsmdSX6YZE+SdyU5K8m9SZ5o2zPb2CS5KcneJI8muWi8tyBJOtywM/fPAd+sqrcBFwB7gG3A7qpaD+xuxwCXA+vb11bg5pF2LEla0pLhnuQNwLuBWwGq6ldV9TywCdjZhu0Ermr7m4DbasEDwOok5428c0nSUQ0zc38zMA98IcnDSW5JcgZwblUdAGjbc9r4NcC+Ra+fa7WXSbI1yWyS2fn5+WXdhCTp5YYJ91XARcDNVXUh8At+uwQzSAbU6ohC1Y6qmqmqmampqaGalSQNZ5hwnwPmqurBdnwnC2H/zKHllrY9uGj8ukWvXwvsH027kqRhLBnuVfVjYF+St7bSRuAHwC5gc6ttBu5q+7uAa9u7ZjYALxxavpEknRirhhz3YeD2JKcDTwLXsfCD4Y4kW4Cngavb2HuAK4C9wIttrCTpBBoq3KvqEWBmwKmNA8YWcP0y+5IkLYOfUJWkDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0aKtyTPJXk+0keSTLbamcluTfJE217ZqsnyU1J9iZ5NMlF47wBSdKRjmXm/sdV9c6qmmnH24DdVbUe2N2OAS4H1revrcDNo2pWkjSc5SzLbAJ2tv2dwFWL6rfVggeA1UnOW8Z1JEnHaNhwL+BbSR5KsrXVzq2qAwBte06rrwH2LXrtXKu9TJKtSWaTzM7Pzx9f95KkgVYNOe6Sqtqf5Bzg3iQ/fIWxGVCrIwpVO4AdADMzM0ecH9b0truP96WS1K2hZu5Vtb9tDwJfBy4Gnjm03NK2B9vwOWDdopevBfaPqmFJ0tKWDPckZyT53UP7wJ8CjwG7gM1t2Gbgrra/C7i2vWtmA/DCoeUbSdKJMcyyzLnA15McGv9PVfXNJN8F7kiyBXgauLqNvwe4AtgLvAhcN/KuJUmvaMlwr6ongQsG1P8b2DigXsD1I+lOknRc/ISqJHXIcJekDg37VkitIJN8++dT26+c2LUlDc+ZuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktShocM9yWlJHk7yjXZ8fpIHkzyR5CtJTm/1V7fjve389HhalyQdzbHM3G8A9iw6/jRwY1WtB54DtrT6FuC5qnoLcGMbJ0k6gYYK9yRrgSuBW9pxgEuBO9uQncBVbX9TO6ad39jGS5JOkGFn7p8FPg78uh2/CXi+ql5qx3PAmra/BtgH0M6/0Ma/TJKtSWaTzM7Pzx9n+5KkQZYM9yTvAw5W1UOLywOG1hDnfluo2lFVM1U1MzU1NVSzkqThrBpizCXA+5NcAbwGeAMLM/nVSVa12flaYH8bPwesA+aSrALeCDw78s4lSUe15My9qj5RVWurahq4Brivqv4MuB/4QBu2Gbir7e9qx7Tz91XVETN3SdL4LOd97n8NfDTJXhbW1G9t9VuBN7X6R4Fty2tRknSshlmW+Y2q+jbw7bb/JHDxgDG/BK4eQW+SpOPkJ1QlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHlgz3JK9J8q9J/i3J40n+ptXPT/JgkieSfCXJ6a3+6na8t52fHu8tSJION8zM/X+AS6vqAuCdwGVJNgCfBm6sqvXAc8CWNn4L8FxVvQW4sY2TJJ1AS4Z7Lfh5O3xV+yrgUuDOVt8JXNX2N7Vj2vmNSTKyjiVJSxpqzT3JaUkeAQ4C9wL/DjxfVS+1IXPAmra/BtgH0M6/ALxpwPfcmmQ2yez8/Pzy7kKS9DJDhXtV/V9VvRNYC1wMvH3QsLYdNEuvIwpVO6pqpqpmpqamhu1XkjSEY3q3TFU9D3wb2ACsTrKqnVoL7G/7c8A6gHb+jcCzo2hWkjScYd4tM5Vkddt/LfBeYA9wP/CBNmwzcFfb39WOaefvq6ojZu6SpPFZtfQQzgN2JjmNhR8Gd1TVN5L8APhykr8FHgZubeNvBf4hyV4WZuzXjKFvSdIrWDLcq+pR4MIB9SdZWH8/vP5L4OqRdCdJOi5+QlWSOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1KFh/icm6Temt909kes+tf3KiVxXOlk5c5ekDhnuktQhw12SOmS4S1KHDHdJ6tCS4Z5kXZL7k+xJ8niSG1r9rCT3Jnmibc9s9SS5KcneJI8muWjcNyFJerlhZu4vAR+rqrcDG4Drk7wD2Absrqr1wO52DHA5sL59bQVuHnnXkqRXtGS4V9WBqvpe2/8ZsAdYA2wCdrZhO4Gr2v4m4LZa8ACwOsl5I+9cknRUx7TmnmQauBB4EDi3qg7Awg8A4Jw2bA2wb9HL5lrt8O+1Nclsktn5+flj71ySdFRDh3uS1wNfBT5SVT99paEDanVEoWpHVc1U1czU1NSwbUiShjBUuCd5FQvBfntVfa2Vnzm03NK2B1t9Dli36OVrgf2jaVeSNIxh3i0T4FZgT1V9ZtGpXcDmtr8ZuGtR/dr2rpkNwAuHlm8kSSfGMA8OuwT4EPD9JI+02ieB7cAdSbYATwNXt3P3AFcAe4EXgetG2rEkaUlLhntV/QuD19EBNg4YX8D1y+xLkrQMfkJVkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOjTMf7MnTdz0trsnct2ntl85ketKy+XMXZI6ZLhLUocMd0nqkOEuSR1aMtyTfD7JwSSPLaqdleTeJE+07ZmtniQ3Jdmb5NEkF42zeUnSYMPM3L8IXHZYbRuwu6rWA7vbMcDlwPr2tRW4eTRtSpKOxZLhXlXfAZ49rLwJ2Nn2dwJXLarfVgseAFYnOW9UzUqShnO8a+7nVtUBgLY9p9XXAPsWjZtrNUnSCTTqX6hmQK0GDky2JplNMjs/Pz/iNiTp1Ha84f7MoeWWtj3Y6nPAukXj1gL7B32DqtpRVTNVNTM1NXWcbUiSBjnecN8FbG77m4G7FtWvbe+a2QC8cGj5RpJ04iz5bJkkXwLeA5ydZA74FLAduCPJFuBp4Oo2/B7gCmAv8CJw3Rh6liQtYclwr6oPHuXUxgFjC7h+uU1JkpbHT6hKUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHlvyEqnQqm95298Su/dT2Kyd2bZ38nLlLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOuSHmKQValIfoPLDU31w5i5JHTLcJalDhrskdcg1d0kv48PS+jCWmXuSy5L8KMneJNvGcQ1J0tGNfOae5DTg74E/AeaA7ybZVVU/GPW1JPXFdwiNzjiWZS4G9lbVkwBJvgxsAgx3SStSj0tR4wj3NcC+RcdzwB8ePijJVmBrO/x5kh+9wvc8G/jJyDpcuU6V+wTvtUenyn3CCO81n17Wy3//aCfGEe4ZUKsjClU7gB1DfcNktqpmltvYSneq3Cd4rz06Ve4TTo57HccvVOeAdYuO1wL7x3AdSdJRjCPcvwusT3J+ktOBa4BdY7iOJOkoRr4sU1UvJfkL4J+B04DPV9Xjy/y2Qy3fdOBUuU/wXnt0qtwnnAT3mqojlsMlSSc5Hz8gSR0y3CWpQys63E+VxxgkWZfk/iR7kjye5IZJ9zROSU5L8nCSb0y6l3FKsjrJnUl+2P5s3zXpnsYlyV+1v7uPJflSktdMuqdRSfL5JAeTPLaodlaSe5M80bZnTrLHQVZsuC96jMHlwDuADyZ5x2S7GpuXgI9V1duBDcD1Hd8rwA3Ankk3cQJ8DvhmVb0NuIBO7znJGuAvgZmq+gMW3khxzWS7GqkvApcdVtsG7K6q9cDudryirNhwZ9FjDKrqV8Chxxh0p6oOVNX32v7PWAiBNZPtajySrAWuBG6ZdC/jlOQNwLuBWwGq6ldV9fxkuxqrVcBrk6wCXkdHn22pqu8Azx5W3gTsbPs7gatOaFNDWMnhPugxBl0G3mJJpoELgQcn28nYfBb4OPDrSTcyZm8G5oEvtCWoW5KcMemmxqGq/gv4O+Bp4ADwQlV9a7Jdjd25VXUAFiZnwDkT7ucIKznch3qMQU+SvB74KvCRqvrppPsZtSTvAw5W1UOT7uUEWAVcBNxcVRcCv2AF/tN9FNp68ybgfOD3gDOS/Plku9JKDvdT6jEGSV7FQrDfXlVfm3Q/Y3IJ8P4kT7GwzHZpkn+cbEtjMwfMVdWhf4HdyULY9+i9wH9U1XxV/S/wNeCPJtzTuD2T5DyAtj044X6OsJLD/ZR5jEGSsLA2u6eqPjPpfsalqj5RVWurapqFP8/7qqrLGV5V/RjYl+StrbSRfh97/TSwIcnr2t/ljXT6y+NFdgGb2/5m4K4J9jLQiv1v9sb0GIOV6hLgQ8D3kzzSap+sqnsm2JOW78PA7W1y8iRw3YT7GYuqejDJncD3WHjn18OcBB/PH1aSLwHvAc5OMgd8CtgO3JFkCws/3K6eXIeD+fgBSerQSl6WkSQdJ8NdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdej/AQuM5YM6Fi1jAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "_=plt.hist(preds_df2[(preds_df2['actual']==1) & (preds_df2['pred']==False)]['gap1'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Elapsed time (gap1) distribution for False Positive"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAASXUlEQVR4nO3db4xd9X3n8fcnGNJAm5g/A2Jta50qVrarSiV0ROkiRV2cVgGi2A+CStQGC3nlPmDbpFRq3T6JKu0DIq1KFmnFysLpmt00lJIgrIRmiwxRNw+gGRMKSZwsE5bgqSmehj8pRWmW9rsP7m/KYM+f65m5vjO/vF/S1Tnne373nu+1rI+Pf/fce1JVSJL68rZxNyBJWnuGuyR1yHCXpA4Z7pLUIcNdkjq0adwNAFxyySW1ffv2cbchSRvK0aNH/66qJhbaty7Cffv27UxNTY27DUnaUJJ8b7F9TstIUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KH1sU3VDeq7fu/NJbjPnf7DWM5rqSNwzN3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1KGhLoVM8tvAfwAKeBq4BbgcuBe4CHgC+FhV/SjJ24F7gJ8Hvg/8alU9t/atD4zrckRJWs+WPXNPsgX4LWCyqn4WOAe4CfgUcEdV7QBeBva2p+wFXq6q9wB3tHGSpLNo2GmZTcA7kmwCzgdeAK4F7m/7DwG72/qutk3bvzNJ1qZdSdIwlg33qvob4D8DzzMI9VeBo8ArVfVGGzYDbGnrW4Dj7blvtPEXn/q6SfYlmUoyNTs7u9r3IUmaZ5hpmQsZnI2/G/hXwAXAdQsMrbmnLLHvzULVgaqarKrJiYkFb94tSVqhYaZlPgD836qarar/B3wB+HfA5jZNA7AVONHWZ4BtAG3/u4CX1rRrSdKShgn354Grk5zf5s53At8CHgU+0sbsAR5s64fbNm3/I1V12pm7JGl0hplzf5zBB6NPMLgM8m3AAeD3gNuSTDOYUz/YnnIQuLjVbwP2j6BvSdIShrrOvao+CXzylPKzwFULjP0hcOPqW5MkrZTfUJWkDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdWiYG2S/N8mT8x4/SPKJJBcleTjJM215YRufJHcmmU7yVJIrR/82JEnzDXObve9U1RVVdQXw88DrwAMMbp93pKp2AEd483Z61wE72mMfcNcoGpckLe5Mp2V2At+tqu8Bu4BDrX4I2N3WdwH31MBjwOYkl69Jt5KkoZxpuN8EfK6tX1ZVLwC05aWtvgU4Pu85M632Fkn2JZlKMjU7O3uGbUiSljJ0uCc5D/gw8GfLDV2gVqcVqg5U1WRVTU5MTAzbhiRpCGdy5n4d8ERVvdi2X5ybbmnLk60+A2yb97ytwInVNipJGt6ZhPtHeXNKBuAwsKet7wEenFe/uV01czXw6tz0jSTp7Ng0zKAk5wO/DPzGvPLtwH1J9gLPAze2+kPA9cA0gytrblmzbiVJQxkq3KvqdeDiU2rfZ3D1zKljC7h1TbqTJK2I31CVpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ0OFe5LNSe5P8u0kx5L8YpKLkjyc5Jm2vLCNTZI7k0wneSrJlaN9C5KkUw175v5fgC9X1b8Bfg44BuwHjlTVDuBI24bBvVZ3tMc+4K417ViStKxlwz3JO4H3AwcBqupHVfUKsAs41IYdAna39V3APTXwGLB57kbakqSzY5gz958GZoE/TvL1JHcnuQC4bO7G1215aRu/BTg+7/kzrfYWSfYlmUoyNTs7u6o3IUl6q2HCfRNwJXBXVb0P+AfenIJZSBao1WmFqgNVNVlVkxMTE0M1K0kazjDhPgPMVNXjbft+BmH/4tx0S1uenDd+27znbwVOrE27kqRhLBvuVfW3wPEk722lncC3gMPAnlbbAzzY1g8DN7erZq4GXp2bvpEknR2bhhz3m8Bnk5wHPAvcwuAfhvuS7AWeB25sYx8CrgemgdfbWEnSWTRUuFfVk8DkArt2LjC2gFtX2ZckaRX8hqokdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUNDhXuS55I8neTJJFOtdlGSh5M805YXtnqS3JlkOslTSa4c5RuQJJ3uTM7c/31VXVFVc3dk2g8cqaodwJG2DXAdsKM99gF3rVWzkqThrGZaZhdwqK0fAnbPq99TA48Bm5NcvorjSJLO0LDhXsBfJDmaZF+rXVZVLwC05aWtvgU4Pu+5M632Fkn2JZlKMjU7O7uy7iVJCxrqBtnANVV1IsmlwMNJvr3E2CxQq9MKVQeAAwCTk5On7ZckrdxQZ+5VdaItTwIPAFcBL85Nt7TlyTZ8Btg27+lbgRNr1bAkaXnLhnuSC5L81Nw68CvAN4DDwJ42bA/wYFs/DNzcrpq5Gnh1bvpGknR2DDMtcxnwQJK58X9SVV9O8jXgviR7geeBG9v4h4DrgWngdeCWNe9akrSkZcO9qp4Ffm6B+veBnQvUC7h1TbqTJK2I31CVpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ0OHe5Jzknw9yRfb9ruTPJ7kmSR/muS8Vn97255u+7ePpnVJ0mLO5Mz948CxedufAu6oqh3Ay8DeVt8LvFxV7wHuaOMkSWfRUOGeZCtwA3B32w5wLXB/G3II2N3Wd7Vt2v6dbbwk6SwZ9sz908DvAv/cti8GXqmqN9r2DLClrW8BjgO0/a+28ZKks2TZcE/yIeBkVR2dX15gaA2xb/7r7ksylWRqdnZ2qGYlScMZ5sz9GuDDSZ4D7mUwHfNpYHOSuRtsbwVOtPUZYBtA2/8u4KVTX7SqDlTVZFVNTkxMrOpNSJLeatlwr6rfr6qtVbUduAl4pKp+DXgU+Egbtgd4sK0fbtu0/Y9U1Wln7pKk0VnNde6/B9yWZJrBnPrBVj8IXNzqtwH7V9eiJOlMbVp+yJuq6ivAV9r6s8BVC4z5IXDjGvQmSVohv6EqSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SerQMDfI/okkf5Xkr5N8M8kftvq7kzye5Jkkf5rkvFZ/e9uebvu3j/YtSJJONcydmP4RuLaqXktyLvDVJH/O4BZ6d1TVvUn+G7AXuKstX66q9yS5CfgU8Ksj6v/H0vb9XxrbsZ+7/YaxHVvS8Ia5QXZV1Wtt89z2KOBa4P5WPwTsbuu72jZt/84kWbOOJUnLGmrOPck5SZ4ETgIPA98FXqmqN9qQGWBLW98CHAdo+19lcAPtU19zX5KpJFOzs7OrexeSpLcYKtyr6p+q6gpgK4ObYv/MQsPacqGz9DqtUHWgqiaranJiYmLYfiVJQzijq2Wq6hXgK8DVwOYkc3P2W4ETbX0G2AbQ9r8LeGktmpUkDWeYq2Umkmxu6+8APgAcAx4FPtKG7QEebOuH2zZt/yNVddqZuyRpdIa5WuZy4FCScxj8Y3BfVX0xybeAe5P8J+DrwME2/iDwP5JMMzhjv2kEfUuSlrBsuFfVU8D7Fqg/y2D+/dT6D4Eb16Q7SdKK+A1VSeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHhrnN3rYkjyY5luSbST7e6hcleTjJM215YasnyZ1JppM8leTKUb8JSdJbDXPm/gbwO1X1MwxujH1rkn8L7AeOVNUO4EjbBrgO2NEe+4C71rxrSdKSlg33qnqhqp5o63/P4ObYW4BdwKE27BCwu63vAu6pgceAzUkuX/POJUmLOqM59yTbGdxP9XHgsqp6AQb/AACXtmFbgOPznjbTaqe+1r4kU0mmZmdnz7xzSdKihg73JD8JfB74RFX9YKmhC9TqtELVgaqarKrJiYmJYduQJA1hqHBPci6DYP9sVX2hlV+cm25py5OtPgNsm/f0rcCJtWlXkjSMYa6WCXAQOFZVfzRv12FgT1vfAzw4r35zu2rmauDVuekbSdLZsWmIMdcAHwOeTvJkq/0BcDtwX5K9wPPAjW3fQ8D1wDTwOnDLmnYsSVrWsuFeVV9l4Xl0gJ0LjC/g1lX2JUlaBb+hKkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDw/y2jPQvtu//0liO+9ztN4zluNJG5Zm7JHXIcJekDhnuktQhw12SOjTMnZg+k+Rkkm/Mq12U5OEkz7Tlha2eJHcmmU7yVJIrR9m8JGlhw5y5/3fgg6fU9gNHqmoHcKRtA1wH7GiPfcBda9OmJOlMLBvuVfWXwEunlHcBh9r6IWD3vPo9NfAYsHnuJtqSpLNnpXPul83d9LotL231LcDxeeNmWu00SfYlmUoyNTs7u8I2JEkLWesPVBe612otNLCqDlTVZFVNTkxMrHEbkvTjbaXh/uLcdEtbnmz1GWDbvHFbgRMrb0+StBIrDffDwJ62vgd4cF795nbVzNXAq3PTN5Kks2fZ35ZJ8jngl4BLkswAnwRuB+5Lshd4HrixDX8IuB6YBl4HbhlBz5KkZSwb7lX10UV27VxgbAG3rrYpSdLq+A1VSeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI65A2ytSF4Y27pzHjmLkkdMtwlqUOGuyR1yHCXpA75gaq0hHF9kAt+mKvV8cxdkjpkuEtShwx3SerQSMI9yQeTfCfJdJL9oziGJGlxa/6BapJzgP8K/DKDG2Z/LcnhqvrWWh9L6tk4P8wdFz9EXjujuFrmKmC6qp4FSHIvsAsw3CUtyZ+ZWDujCPctwPF52zPAL5w6KMk+YF/bfC3Jd1Z4vEuAv1vhc882ex2NjdQrbKx+fyx6zafWuJPhrMWf7b9ebMcowj0L1Oq0QtUB4MCqD5ZMVdXkal/nbLDX0dhIvcLG6tdeR2fU/Y7iA9UZYNu87a3AiREcR5K0iFGE+9eAHUneneQ84Cbg8AiOI0laxJpPy1TVG0n+I/C/gHOAz1TVN9f6OPOsemrnLLLX0dhIvcLG6tdeR2ek/abqtOlwSdIG5zdUJalDhrskdWjDhvtG+omDJJ9JcjLJN8bdy3KSbEvyaJJjSb6Z5OPj7mkxSX4iyV8l+evW6x+Ou6flJDknydeTfHHcvSwnyXNJnk7yZJKpcfezlCSbk9yf5Nvt7+4vjrunhSR5b/vznHv8IMknRnKsjTjn3n7i4P8w7ycOgI+u1584SPJ+4DXgnqr62XH3s5QklwOXV9UTSX4KOArsXo9/tkkCXFBVryU5F/gq8PGqemzMrS0qyW3AJPDOqvrQuPtZSpLngMmqWvdfYkpyCPjfVXV3u0rv/Kp6Zdx9LaXl2N8Av1BV31vr19+oZ+7/8hMHVfUjYO4nDtalqvpL4KVx9zGMqnqhqp5o638PHGPwreN1pwZea5vntse6PVtJshW4Abh73L30JMk7gfcDBwGq6kfrPdibncB3RxHssHHDfaGfOFiXAbSRJdkOvA94fLydLK5NczwJnAQerqp12yvwaeB3gX8edyNDKuAvkhxtPxeyXv00MAv8cZvyujvJBeNuagg3AZ8b1Ytv1HAf6icOtHJJfhL4PPCJqvrBuPtZTFX9U1VdweCb0FclWZfTXkk+BJysqqPj7uUMXFNVVwLXAbe26cX1aBNwJXBXVb0P+AdgvX8Odx7wYeDPRnWMjRru/sTBCLX5688Dn62qL4y7n2G0/4Z/BfjgmFtZzDXAh9s89r3AtUn+53hbWlpVnWjLk8ADDKZD16MZYGbe/9ruZxD269l1wBNV9eKoDrBRw92fOBiR9iHlQeBYVf3RuPtZSpKJJJvb+juADwDfHm9XC6uq36+qrVW1ncHf10eq6tfH3NaiklzQPlCnTXH8CrAur/aqqr8Fjid5byvtZP3/xPhHGeGUDIzmVyFHbgw/cbAqST4H/BJwSZIZ4JNVdXC8XS3qGuBjwNNtLhvgD6rqoTH2tJjLgUPtqoO3AfdV1bq/xHCDuAx4YPBvPZuAP6mqL4+3pSX9JvDZdrL3LHDLmPtZVJLzGVzp9xsjPc5GvBRSkrS0jTotI0laguEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOvT/AfysCd7rS9gFAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "_=plt.hist(preds_df2[(preds_df2['actual']==0) & (preds_df2['pred']==True)]['gap1'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}