FeatureEngineering.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Starbucks Capstone Challenge\n",
    "# Feature Engineering\n",
    "\n",
    "This notebook aims to generate features by preparing the data to correspond to the problem stated and feed the neural networks.  \n",
    "The transcription records must be structured and labeled as an appropriate offer or not.\n",
    "\n",
    "Feature Engineering is the process of determining which features might be useful in training a model, and then converting raw data from log files and other sources into said features."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Import all the necessary libraries\n",
    "import os\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import torch\n",
    "from torch.utils.data import DataLoader, TensorDataset\n",
    "from sklearn.preprocessing import quantile_transform, scale, robust_scale"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Global definitions\n",
    "\n",
    "data_dir = 'data'\n",
    "\n",
    "pd.set_option('display.precision', 2)\n",
    "pd.set_option('display.max_colwidth', None)\n",
    "pd.set_option('display.max_columns', None)\n",
    "\n",
    "portfolio_data_path = os.path.join(data_dir, 'portfolio.json')\n",
    "profile_data_path = os.path.join(data_dir, 'profile.json')\n",
    "transcript_data_path = os.path.join(data_dir, 'transcript.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "## global functions\n",
    "\n",
    "def load_dataframe(data_path):\n",
    "    \"\"\"Create a dataframe from a json file\"\"\"\n",
    "    return pd.read_json(data_path, orient='records', lines=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Portfolio dataset\n",
    "Dataset containing information about the offers which can be sent to customers."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Overview"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>reward</th>\n",
       "      <th>channels</th>\n",
       "      <th>difficulty</th>\n",
       "      <th>duration</th>\n",
       "      <th>offer_type</th>\n",
       "      <th>id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10</td>\n",
       "      <td>[email, mobile, social]</td>\n",
       "      <td>10</td>\n",
       "      <td>7</td>\n",
       "      <td>bogo</td>\n",
       "      <td>ae264e3637204a6fb9bb56bc8210ddfd</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10</td>\n",
       "      <td>[web, email, mobile, social]</td>\n",
       "      <td>10</td>\n",
       "      <td>5</td>\n",
       "      <td>bogo</td>\n",
       "      <td>4d5c57ea9a6940dd891ad53e9dbe8da0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>[web, email, mobile]</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>informational</td>\n",
       "      <td>3f207df678b143eea3cee63160fa8bed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5</td>\n",
       "      <td>[web, email, mobile]</td>\n",
       "      <td>5</td>\n",
       "      <td>7</td>\n",
       "      <td>bogo</td>\n",
       "      <td>9b98b8c7a33c4b65b9aebfe6a799e6d9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>[web, email]</td>\n",
       "      <td>20</td>\n",
       "      <td>10</td>\n",
       "      <td>discount</td>\n",
       "      <td>0b1e1539f2cc45b7b9fa7c272da2e1d7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   reward                      channels  difficulty  duration     offer_type  \\\n",
       "0      10       [email, mobile, social]          10         7           bogo   \n",
       "1      10  [web, email, mobile, social]          10         5           bogo   \n",
       "2       0          [web, email, mobile]           0         4  informational   \n",
       "3       5          [web, email, mobile]           5         7           bogo   \n",
       "4       5                  [web, email]          20        10       discount   \n",
       "\n",
       "                                 id  \n",
       "0  ae264e3637204a6fb9bb56bc8210ddfd  \n",
       "1  4d5c57ea9a6940dd891ad53e9dbe8da0  \n",
       "2  3f207df678b143eea3cee63160fa8bed  \n",
       "3  9b98b8c7a33c4b65b9aebfe6a799e6d9  \n",
       "4  0b1e1539f2cc45b7b9fa7c272da2e1d7  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "## Load dataset\n",
    "portfolio_df = load_dataframe(portfolio_data_path)\n",
    "display(portfolio_df.head())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Transformation\n",
    "As we can see **offer type** is a categorical feature that could be mapped as one hot encoding.  \n",
    "\n",
    "**Channels** are categorical feature as well, but values can assume more than one category. Its values may be converted into individual features.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>reward</th>\n",
       "      <th>difficulty</th>\n",
       "      <th>duration</th>\n",
       "      <th>bogo</th>\n",
       "      <th>discount</th>\n",
       "      <th>informational</th>\n",
       "      <th>email</th>\n",
       "      <th>mobile</th>\n",
       "      <th>social</th>\n",
       "      <th>web</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ae264e3637204a6fb9bb56bc8210ddfd</th>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4d5c57ea9a6940dd891ad53e9dbe8da0</th>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3f207df678b143eea3cee63160fa8bed</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9b98b8c7a33c4b65b9aebfe6a799e6d9</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0b1e1539f2cc45b7b9fa7c272da2e1d7</th>\n",
       "      <td>5</td>\n",
       "      <td>20</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2298d6c36e964ae4a3e7e9706d1fb8c2</th>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fafdcd668e3743c1bb461111dcafc2a4</th>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5a8bc65990b245e5a138643cd4eb9837</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>f19421c1d4aa40978ebb69ca19b0e20d</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2906b810c7d4411798c6938adc9daaa5</th>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  reward  difficulty  duration  bogo  \\\n",
       "id                                                                     \n",
       "ae264e3637204a6fb9bb56bc8210ddfd      10          10         7     1   \n",
       "4d5c57ea9a6940dd891ad53e9dbe8da0      10          10         5     1   \n",
       "3f207df678b143eea3cee63160fa8bed       0           0         4     0   \n",
       "9b98b8c7a33c4b65b9aebfe6a799e6d9       5           5         7     1   \n",
       "0b1e1539f2cc45b7b9fa7c272da2e1d7       5          20        10     0   \n",
       "2298d6c36e964ae4a3e7e9706d1fb8c2       3           7         7     0   \n",
       "fafdcd668e3743c1bb461111dcafc2a4       2          10        10     0   \n",
       "5a8bc65990b245e5a138643cd4eb9837       0           0         3     0   \n",
       "f19421c1d4aa40978ebb69ca19b0e20d       5           5         5     1   \n",
       "2906b810c7d4411798c6938adc9daaa5       2          10         7     0   \n",
       "\n",
       "                                  discount  informational  email  mobile  \\\n",
       "id                                                                         \n",
       "ae264e3637204a6fb9bb56bc8210ddfd         0              0    1.0     1.0   \n",
       "4d5c57ea9a6940dd891ad53e9dbe8da0         0              0    1.0     1.0   \n",
       "3f207df678b143eea3cee63160fa8bed         0              1    1.0     1.0   \n",
       "9b98b8c7a33c4b65b9aebfe6a799e6d9         0              0    1.0     1.0   \n",
       "0b1e1539f2cc45b7b9fa7c272da2e1d7         1              0    1.0     0.0   \n",
       "2298d6c36e964ae4a3e7e9706d1fb8c2         1              0    1.0     1.0   \n",
       "fafdcd668e3743c1bb461111dcafc2a4         1              0    1.0     1.0   \n",
       "5a8bc65990b245e5a138643cd4eb9837         0              1    1.0     1.0   \n",
       "f19421c1d4aa40978ebb69ca19b0e20d         0              0    1.0     1.0   \n",
       "2906b810c7d4411798c6938adc9daaa5         1              0    1.0     1.0   \n",
       "\n",
       "                                  social  web  \n",
       "id                                             \n",
       "ae264e3637204a6fb9bb56bc8210ddfd     1.0  0.0  \n",
       "4d5c57ea9a6940dd891ad53e9dbe8da0     1.0  1.0  \n",
       "3f207df678b143eea3cee63160fa8bed     0.0  1.0  \n",
       "9b98b8c7a33c4b65b9aebfe6a799e6d9     0.0  1.0  \n",
       "0b1e1539f2cc45b7b9fa7c272da2e1d7     0.0  1.0  \n",
       "2298d6c36e964ae4a3e7e9706d1fb8c2     1.0  1.0  \n",
       "fafdcd668e3743c1bb461111dcafc2a4     1.0  1.0  \n",
       "5a8bc65990b245e5a138643cd4eb9837     1.0  0.0  \n",
       "f19421c1d4aa40978ebb69ca19b0e20d     1.0  1.0  \n",
       "2906b810c7d4411798c6938adc9daaa5     0.0  1.0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "## Set id as index\n",
    "portfolio_df.set_index(keys='id', verify_integrity=True, inplace=True)\n",
    "\n",
    "## Make offer_type one hot encoded\n",
    "portfolio_df = portfolio_df.join(\n",
    "    pd.get_dummies(portfolio_df.pop('offer_type')))\n",
    "\n",
    "## Transform channels in distinct features\n",
    "channels_df = pd.DataFrame(portfolio_df.pop('channels'))\n",
    "channels_df = channels_df.explode('channels')\n",
    "channels_df = channels_df.assign(value=lambda x: 1)\n",
    "channels_df = channels_df.pivot(columns='channels', values='value')\n",
    "channels_df.fillna(value=0, inplace=True)\n",
    "portfolio_df = portfolio_df.join(channels_df)\n",
    "channels_df = None\n",
    "\n",
    "## print the result\n",
    "display(portfolio_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "</style><table id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26\" ><caption>Dataset description</caption><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >reward</th>        <th class=\"col_heading level0 col1\" >difficulty</th>        <th class=\"col_heading level0 col2\" >duration</th>        <th class=\"col_heading level0 col3\" >bogo</th>        <th class=\"col_heading level0 col4\" >discount</th>        <th class=\"col_heading level0 col5\" >informational</th>        <th class=\"col_heading level0 col6\" >mobile</th>        <th class=\"col_heading level0 col7\" >social</th>        <th class=\"col_heading level0 col8\" >web</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26level0_row0\" class=\"row_heading level0 row0\" >count</th>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row0_col0\" class=\"data row0 col0\" >10.00</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row0_col1\" class=\"data row0 col1\" >10.00</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row0_col2\" class=\"data row0 col2\" >10.00</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row0_col3\" class=\"data row0 col3\" >10.00</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row0_col4\" class=\"data row0 col4\" >10.00</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row0_col5\" class=\"data row0 col5\" >10.00</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row0_col6\" class=\"data row0 col6\" >10.00</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row0_col7\" class=\"data row0 col7\" >10.00</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row0_col8\" class=\"data row0 col8\" >10.00</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26level0_row1\" class=\"row_heading level0 row1\" >mean</th>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row1_col0\" class=\"data row1 col0\" >4.20</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row1_col1\" class=\"data row1 col1\" >7.70</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row1_col2\" class=\"data row1 col2\" >6.50</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row1_col3\" class=\"data row1 col3\" >0.40</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row1_col4\" class=\"data row1 col4\" >0.40</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row1_col5\" class=\"data row1 col5\" >0.20</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row1_col6\" class=\"data row1 col6\" >0.90</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row1_col7\" class=\"data row1 col7\" >0.60</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row1_col8\" class=\"data row1 col8\" >0.80</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26level0_row2\" class=\"row_heading level0 row2\" >std</th>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row2_col0\" class=\"data row2 col0\" >3.58</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row2_col1\" class=\"data row2 col1\" >5.83</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row2_col2\" class=\"data row2 col2\" >2.32</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row2_col3\" class=\"data row2 col3\" >0.52</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row2_col4\" class=\"data row2 col4\" >0.52</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row2_col5\" class=\"data row2 col5\" >0.42</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row2_col6\" class=\"data row2 col6\" >0.32</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row2_col7\" class=\"data row2 col7\" >0.52</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row2_col8\" class=\"data row2 col8\" >0.42</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26level0_row3\" class=\"row_heading level0 row3\" >min</th>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row3_col0\" class=\"data row3 col0\" >0.00</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row3_col1\" class=\"data row3 col1\" >0.00</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row3_col2\" class=\"data row3 col2\" >3.00</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row3_col3\" class=\"data row3 col3\" >0.00</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row3_col4\" class=\"data row3 col4\" >0.00</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row3_col5\" class=\"data row3 col5\" >0.00</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row3_col6\" class=\"data row3 col6\" >0.00</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row3_col7\" class=\"data row3 col7\" >0.00</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row3_col8\" class=\"data row3 col8\" >0.00</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26level0_row4\" class=\"row_heading level0 row4\" >25%</th>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row4_col0\" class=\"data row4 col0\" >2.00</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row4_col1\" class=\"data row4 col1\" >5.00</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row4_col2\" class=\"data row4 col2\" >5.00</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row4_col3\" class=\"data row4 col3\" >0.00</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row4_col4\" class=\"data row4 col4\" >0.00</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row4_col5\" class=\"data row4 col5\" >0.00</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row4_col6\" class=\"data row4 col6\" >1.00</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row4_col7\" class=\"data row4 col7\" >0.00</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row4_col8\" class=\"data row4 col8\" >1.00</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26level0_row5\" class=\"row_heading level0 row5\" >50%</th>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row5_col0\" class=\"data row5 col0\" >4.00</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row5_col1\" class=\"data row5 col1\" >8.50</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row5_col2\" class=\"data row5 col2\" >7.00</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row5_col3\" class=\"data row5 col3\" >0.00</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row5_col4\" class=\"data row5 col4\" >0.00</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row5_col5\" class=\"data row5 col5\" >0.00</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row5_col6\" class=\"data row5 col6\" >1.00</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row5_col7\" class=\"data row5 col7\" >1.00</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row5_col8\" class=\"data row5 col8\" >1.00</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26level0_row6\" class=\"row_heading level0 row6\" >75%</th>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row6_col0\" class=\"data row6 col0\" >5.00</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row6_col1\" class=\"data row6 col1\" >10.00</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row6_col2\" class=\"data row6 col2\" >7.00</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row6_col3\" class=\"data row6 col3\" >1.00</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row6_col4\" class=\"data row6 col4\" >1.00</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row6_col5\" class=\"data row6 col5\" >0.00</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row6_col6\" class=\"data row6 col6\" >1.00</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row6_col7\" class=\"data row6 col7\" >1.00</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row6_col8\" class=\"data row6 col8\" >1.00</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26level0_row7\" class=\"row_heading level0 row7\" >max</th>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row7_col0\" class=\"data row7 col0\" >10.00</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row7_col1\" class=\"data row7 col1\" >20.00</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row7_col2\" class=\"data row7 col2\" >10.00</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row7_col3\" class=\"data row7 col3\" >1.00</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row7_col4\" class=\"data row7 col4\" >1.00</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row7_col5\" class=\"data row7 col5\" >1.00</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row7_col6\" class=\"data row7 col6\" >1.00</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row7_col7\" class=\"data row7 col7\" >1.00</td>\n",
       "                        <td id=\"T_445eb624_4b91_11ea_9436_17ddfb4f7b26row7_col8\" class=\"data row7 col8\" >1.00</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7ff711499518>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "</style><table id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26\" ><caption>Pairwise correlation</caption><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >reward</th>        <th class=\"col_heading level0 col1\" >difficulty</th>        <th class=\"col_heading level0 col2\" >duration</th>        <th class=\"col_heading level0 col3\" >bogo</th>        <th class=\"col_heading level0 col4\" >discount</th>        <th class=\"col_heading level0 col5\" >informational</th>        <th class=\"col_heading level0 col6\" >mobile</th>        <th class=\"col_heading level0 col7\" >social</th>        <th class=\"col_heading level0 col8\" >web</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26level0_row0\" class=\"row_heading level0 row0\" >reward</th>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row0_col0\" class=\"data row0 col0\" >1.00</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row0_col1\" class=\"data row0 col1\" >0.47</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row0_col2\" class=\"data row0 col2\" >0.16</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row0_col3\" class=\"data row0 col3\" >0.79</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row0_col4\" class=\"data row0 col4\" >0.29</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row0_col5\" class=\"data row0 col5\" >0.62</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row0_col6\" class=\"data row0 col6\" >0.08</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row0_col7\" class=\"data row0 col7\" >0.29</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row0_col8\" class=\"data row0 col8\" >0.12</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26level0_row1\" class=\"row_heading level0 row1\" >difficulty</th>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row1_col0\" class=\"data row1 col0\" >0.47</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row1_col1\" class=\"data row1 col1\" >1.00</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row1_col2\" class=\"data row1 col2\" >0.81</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row1_col3\" class=\"data row1 col3\" >0.03</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row1_col4\" class=\"data row1 col4\" >0.60</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row1_col5\" class=\"data row1 col5\" >0.70</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row1_col6\" class=\"data row1 col6\" >0.74</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row1_col7\" class=\"data row1 col7\" >0.15</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row1_col8\" class=\"data row1 col8\" >0.24</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26level0_row2\" class=\"row_heading level0 row2\" >duration</th>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row2_col0\" class=\"data row2 col0\" >0.16</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row2_col1\" class=\"data row2 col1\" >0.81</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row2_col2\" class=\"data row2 col2\" >1.00</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row2_col3\" class=\"data row2 col3\" >0.19</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row2_col4\" class=\"data row2 col4\" >0.74</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row2_col5\" class=\"data row2 col5\" >0.68</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row2_col6\" class=\"data row2 col6\" >0.53</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row2_col7\" class=\"data row2 col7\" >0.19</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row2_col8\" class=\"data row2 col8\" >0.34</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26level0_row3\" class=\"row_heading level0 row3\" >bogo</th>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row3_col0\" class=\"data row3 col0\" >0.79</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row3_col1\" class=\"data row3 col1\" >0.03</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row3_col2\" class=\"data row3 col2\" >0.19</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row3_col3\" class=\"data row3 col3\" >1.00</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row3_col4\" class=\"data row3 col4\" >0.67</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row3_col5\" class=\"data row3 col5\" >0.41</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row3_col6\" class=\"data row3 col6\" >0.27</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row3_col7\" class=\"data row3 col7\" >0.25</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row3_col8\" class=\"data row3 col8\" >0.10</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26level0_row4\" class=\"row_heading level0 row4\" >discount</th>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row4_col0\" class=\"data row4 col0\" >0.29</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row4_col1\" class=\"data row4 col1\" >0.60</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row4_col2\" class=\"data row4 col2\" >0.74</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row4_col3\" class=\"data row4 col3\" >0.67</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row4_col4\" class=\"data row4 col4\" >1.00</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row4_col5\" class=\"data row4 col5\" >0.41</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row4_col6\" class=\"data row4 col6\" >0.41</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row4_col7\" class=\"data row4 col7\" >0.17</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row4_col8\" class=\"data row4 col8\" >0.41</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26level0_row5\" class=\"row_heading level0 row5\" >informational</th>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row5_col0\" class=\"data row5 col0\" >0.62</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row5_col1\" class=\"data row5 col1\" >0.70</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row5_col2\" class=\"data row5 col2\" >0.68</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row5_col3\" class=\"data row5 col3\" >0.41</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row5_col4\" class=\"data row5 col4\" >0.41</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row5_col5\" class=\"data row5 col5\" >1.00</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row5_col6\" class=\"data row5 col6\" >0.17</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row5_col7\" class=\"data row5 col7\" >0.10</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row5_col8\" class=\"data row5 col8\" >0.37</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26level0_row6\" class=\"row_heading level0 row6\" >mobile</th>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row6_col0\" class=\"data row6 col0\" >0.08</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row6_col1\" class=\"data row6 col1\" >0.74</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row6_col2\" class=\"data row6 col2\" >0.53</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row6_col3\" class=\"data row6 col3\" >0.27</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row6_col4\" class=\"data row6 col4\" >0.41</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row6_col5\" class=\"data row6 col5\" >0.17</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row6_col6\" class=\"data row6 col6\" >1.00</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row6_col7\" class=\"data row6 col7\" >0.41</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row6_col8\" class=\"data row6 col8\" >0.17</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26level0_row7\" class=\"row_heading level0 row7\" >social</th>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row7_col0\" class=\"data row7 col0\" >0.29</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row7_col1\" class=\"data row7 col1\" >0.15</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row7_col2\" class=\"data row7 col2\" >0.19</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row7_col3\" class=\"data row7 col3\" >0.25</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row7_col4\" class=\"data row7 col4\" >0.17</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row7_col5\" class=\"data row7 col5\" >0.10</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row7_col6\" class=\"data row7 col6\" >0.41</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row7_col7\" class=\"data row7 col7\" >1.00</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row7_col8\" class=\"data row7 col8\" >0.41</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26level0_row8\" class=\"row_heading level0 row8\" >web</th>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row8_col0\" class=\"data row8 col0\" >0.12</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row8_col1\" class=\"data row8 col1\" >0.24</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row8_col2\" class=\"data row8 col2\" >0.34</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row8_col3\" class=\"data row8 col3\" >0.10</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row8_col4\" class=\"data row8 col4\" >0.41</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row8_col5\" class=\"data row8 col5\" >0.37</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row8_col6\" class=\"data row8 col6\" >0.17</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row8_col7\" class=\"data row8 col7\" >0.41</td>\n",
       "                        <td id=\"T_445f87c0_4b91_11ea_9436_17ddfb4f7b26row8_col8\" class=\"data row8 col8\" >1.00</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7ff710588cc0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "## Filter out email column, since it is not an informative feature\n",
    "# once any offer uses this channel.\n",
    "portfolio_df.drop(columns='email', inplace=True)\n",
    "\n",
    "display(pd.DataFrame(portfolio_df.describe())\n",
    "        .style.set_caption('Dataset description'))\n",
    "\n",
    "display(portfolio_df.corr().abs()\n",
    "        .style.set_caption('Pairwise correlation'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Resulting dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>reward</th>\n",
       "      <th>difficulty</th>\n",
       "      <th>duration</th>\n",
       "      <th>bogo</th>\n",
       "      <th>discount</th>\n",
       "      <th>informational</th>\n",
       "      <th>mobile</th>\n",
       "      <th>social</th>\n",
       "      <th>web</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ae264e3637204a6fb9bb56bc8210ddfd</th>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4d5c57ea9a6940dd891ad53e9dbe8da0</th>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3f207df678b143eea3cee63160fa8bed</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9b98b8c7a33c4b65b9aebfe6a799e6d9</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0b1e1539f2cc45b7b9fa7c272da2e1d7</th>\n",
       "      <td>5</td>\n",
       "      <td>20</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2298d6c36e964ae4a3e7e9706d1fb8c2</th>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fafdcd668e3743c1bb461111dcafc2a4</th>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5a8bc65990b245e5a138643cd4eb9837</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>f19421c1d4aa40978ebb69ca19b0e20d</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2906b810c7d4411798c6938adc9daaa5</th>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  reward  difficulty  duration  bogo  \\\n",
       "id                                                                     \n",
       "ae264e3637204a6fb9bb56bc8210ddfd      10          10         7     1   \n",
       "4d5c57ea9a6940dd891ad53e9dbe8da0      10          10         5     1   \n",
       "3f207df678b143eea3cee63160fa8bed       0           0         4     0   \n",
       "9b98b8c7a33c4b65b9aebfe6a799e6d9       5           5         7     1   \n",
       "0b1e1539f2cc45b7b9fa7c272da2e1d7       5          20        10     0   \n",
       "2298d6c36e964ae4a3e7e9706d1fb8c2       3           7         7     0   \n",
       "fafdcd668e3743c1bb461111dcafc2a4       2          10        10     0   \n",
       "5a8bc65990b245e5a138643cd4eb9837       0           0         3     0   \n",
       "f19421c1d4aa40978ebb69ca19b0e20d       5           5         5     1   \n",
       "2906b810c7d4411798c6938adc9daaa5       2          10         7     0   \n",
       "\n",
       "                                  discount  informational  mobile  social  web  \n",
       "id                                                                              \n",
       "ae264e3637204a6fb9bb56bc8210ddfd         0              0     1.0     1.0  0.0  \n",
       "4d5c57ea9a6940dd891ad53e9dbe8da0         0              0     1.0     1.0  1.0  \n",
       "3f207df678b143eea3cee63160fa8bed         0              1     1.0     0.0  1.0  \n",
       "9b98b8c7a33c4b65b9aebfe6a799e6d9         0              0     1.0     0.0  1.0  \n",
       "0b1e1539f2cc45b7b9fa7c272da2e1d7         1              0     0.0     0.0  1.0  \n",
       "2298d6c36e964ae4a3e7e9706d1fb8c2         1              0     1.0     1.0  1.0  \n",
       "fafdcd668e3743c1bb461111dcafc2a4         1              0     1.0     1.0  1.0  \n",
       "5a8bc65990b245e5a138643cd4eb9837         0              1     1.0     1.0  0.0  \n",
       "f19421c1d4aa40978ebb69ca19b0e20d         0              0     1.0     1.0  1.0  \n",
       "2906b810c7d4411798c6938adc9daaa5         1              0     1.0     0.0  1.0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(portfolio_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Profile data set\n",
    "Dataset containing demographic data for each one of the reward program users."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Overview"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gender</th>\n",
       "      <th>age</th>\n",
       "      <th>id</th>\n",
       "      <th>became_member_on</th>\n",
       "      <th>income</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>None</td>\n",
       "      <td>118</td>\n",
       "      <td>68be06ca386d4c31939f3a4f0e3dd783</td>\n",
       "      <td>20170212</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>F</td>\n",
       "      <td>55</td>\n",
       "      <td>0610b486422d4921ae7d2bf64640c50b</td>\n",
       "      <td>20170715</td>\n",
       "      <td>112000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>None</td>\n",
       "      <td>118</td>\n",
       "      <td>38fe809add3b4fcf9315a9694bb96ff5</td>\n",
       "      <td>20180712</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>F</td>\n",
       "      <td>75</td>\n",
       "      <td>78afa995795e4d85b5d9ceeca43f5fef</td>\n",
       "      <td>20170509</td>\n",
       "      <td>100000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>None</td>\n",
       "      <td>118</td>\n",
       "      <td>a03223e636434f42ac4c3df47e8bac43</td>\n",
       "      <td>20170804</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  gender  age                                id  became_member_on    income\n",
       "0   None  118  68be06ca386d4c31939f3a4f0e3dd783          20170212       NaN\n",
       "1      F   55  0610b486422d4921ae7d2bf64640c50b          20170715  112000.0\n",
       "2   None  118  38fe809add3b4fcf9315a9694bb96ff5          20180712       NaN\n",
       "3      F   75  78afa995795e4d85b5d9ceeca43f5fef          20170509  100000.0\n",
       "4   None  118  a03223e636434f42ac4c3df47e8bac43          20170804       NaN"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "profile_df = load_dataframe(profile_data_path)\n",
    "display(profile_df.head())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Transformation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Altough the users with missing data could be filtered out of the data set, they might have some peculiar behaviour which would be interesting to be analyzed.  \n",
    "Thus, my approach with those record is:  \n",
    "* maintain those records in the data set\n",
    "* mark those users as belonging to a particular class, by creating a new feature for that\n",
    "* since gender is a discrete feature, create another gender category\n",
    "* as age and income are continuous features, fill them with the respective mean values\n",
    "\n",
    "The feature *became_member_on* is a date represented as a string, so that the values is difficulty to compare due to the lack of linearity.  \n",
    "Consider the dates 20171230, 20171231, and 20180101. Altough the distances between them are the same (1 day), if they are considered as numbers the distance becames 1 and 8870, respectively.  \n",
    "Thus, this feature should be transformed to the type long with a continuous representantion. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create new feature to indicate missing values\n",
    "missing_data = profile_df.gender.isna()\n",
    "profile_df = profile_df.assign(missing_data=missing_data.astype(int))\n",
    "\n",
    "# fill gender if a new 'U' (unknown) category\n",
    "profile_df.gender.mask(missing_data, 'U', inplace=True)\n",
    "\n",
    "# Make gender one-hot encoded\n",
    "profile_df = profile_df.join(\n",
    "    pd.get_dummies(profile_df.pop('gender')))\n",
    "\n",
    "# Set id as index\n",
    "profile_df.set_index(keys='id', verify_integrity=True, inplace=True)\n",
    "\n",
    "## Convert data to datetime format\n",
    "profile_df.became_member_on = pd.to_datetime(profile_df.became_member_on, format='%Y%m%d').astype(np.long)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "</style><table id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26\" ><caption>Dataset description</caption><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >age</th>        <th class=\"col_heading level0 col1\" >became_member_on</th>        <th class=\"col_heading level0 col2\" >income</th>        <th class=\"col_heading level0 col3\" >missing_data</th>        <th class=\"col_heading level0 col4\" >F</th>        <th class=\"col_heading level0 col5\" >M</th>        <th class=\"col_heading level0 col6\" >O</th>        <th class=\"col_heading level0 col7\" >U</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26level0_row0\" class=\"row_heading level0 row0\" >count</th>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row0_col0\" class=\"data row0 col0\" >17000.00</td>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row0_col1\" class=\"data row0 col1\" >17000.00</td>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row0_col2\" class=\"data row0 col2\" >14825.00</td>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row0_col3\" class=\"data row0 col3\" >17000.00</td>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row0_col4\" class=\"data row0 col4\" >17000.00</td>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row0_col5\" class=\"data row0 col5\" >17000.00</td>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row0_col6\" class=\"data row0 col6\" >17000.00</td>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row0_col7\" class=\"data row0 col7\" >17000.00</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26level0_row1\" class=\"row_heading level0 row1\" >mean</th>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row1_col0\" class=\"data row1 col0\" >62.53</td>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row1_col1\" class=\"data row1 col1\" >1487855530164706048.00</td>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row1_col2\" class=\"data row1 col2\" >65404.99</td>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row1_col3\" class=\"data row1 col3\" >0.13</td>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row1_col4\" class=\"data row1 col4\" >0.36</td>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row1_col5\" class=\"data row1 col5\" >0.50</td>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row1_col6\" class=\"data row1 col6\" >0.01</td>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row1_col7\" class=\"data row1 col7\" >0.13</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26level0_row2\" class=\"row_heading level0 row2\" >std</th>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row2_col0\" class=\"data row2 col0\" >26.74</td>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row2_col1\" class=\"data row2 col1\" >35529745278667744.00</td>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row2_col2\" class=\"data row2 col2\" >21598.30</td>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row2_col3\" class=\"data row2 col3\" >0.33</td>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row2_col4\" class=\"data row2 col4\" >0.48</td>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row2_col5\" class=\"data row2 col5\" >0.50</td>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row2_col6\" class=\"data row2 col6\" >0.11</td>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row2_col7\" class=\"data row2 col7\" >0.33</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26level0_row3\" class=\"row_heading level0 row3\" >min</th>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row3_col0\" class=\"data row3 col0\" >18.00</td>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row3_col1\" class=\"data row3 col1\" >1375056000000000000.00</td>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row3_col2\" class=\"data row3 col2\" >30000.00</td>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row3_col3\" class=\"data row3 col3\" >0.00</td>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row3_col4\" class=\"data row3 col4\" >0.00</td>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row3_col5\" class=\"data row3 col5\" >0.00</td>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row3_col6\" class=\"data row3 col6\" >0.00</td>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row3_col7\" class=\"data row3 col7\" >0.00</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26level0_row4\" class=\"row_heading level0 row4\" >25%</th>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row4_col0\" class=\"data row4 col0\" >45.00</td>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row4_col1\" class=\"data row4 col1\" >1464220800000000000.00</td>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row4_col2\" class=\"data row4 col2\" >49000.00</td>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row4_col3\" class=\"data row4 col3\" >0.00</td>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row4_col4\" class=\"data row4 col4\" >0.00</td>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row4_col5\" class=\"data row4 col5\" >0.00</td>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row4_col6\" class=\"data row4 col6\" >0.00</td>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row4_col7\" class=\"data row4 col7\" >0.00</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26level0_row5\" class=\"row_heading level0 row5\" >50%</th>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row5_col0\" class=\"data row5 col0\" >58.00</td>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row5_col1\" class=\"data row5 col1\" >1501632000000000000.00</td>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row5_col2\" class=\"data row5 col2\" >64000.00</td>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row5_col3\" class=\"data row5 col3\" >0.00</td>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row5_col4\" class=\"data row5 col4\" >0.00</td>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row5_col5\" class=\"data row5 col5\" >0.00</td>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row5_col6\" class=\"data row5 col6\" >0.00</td>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row5_col7\" class=\"data row5 col7\" >0.00</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26level0_row6\" class=\"row_heading level0 row6\" >75%</th>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row6_col0\" class=\"data row6 col0\" >73.00</td>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row6_col1\" class=\"data row6 col1\" >1514592000000000000.00</td>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row6_col2\" class=\"data row6 col2\" >80000.00</td>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row6_col3\" class=\"data row6 col3\" >0.00</td>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row6_col4\" class=\"data row6 col4\" >1.00</td>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row6_col5\" class=\"data row6 col5\" >1.00</td>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row6_col6\" class=\"data row6 col6\" >0.00</td>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row6_col7\" class=\"data row6 col7\" >0.00</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26level0_row7\" class=\"row_heading level0 row7\" >max</th>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row7_col0\" class=\"data row7 col0\" >118.00</td>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row7_col1\" class=\"data row7 col1\" >1532563200000000000.00</td>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row7_col2\" class=\"data row7 col2\" >120000.00</td>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row7_col3\" class=\"data row7 col3\" >1.00</td>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row7_col4\" class=\"data row7 col4\" >1.00</td>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row7_col5\" class=\"data row7 col5\" >1.00</td>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row7_col6\" class=\"data row7 col6\" >1.00</td>\n",
       "                        <td id=\"T_4477db04_4b91_11ea_9436_17ddfb4f7b26row7_col7\" class=\"data row7 col7\" >1.00</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7ff7114b5cf8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "</style><table id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26\" ><caption>Pairwise correlation</caption><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >age</th>        <th class=\"col_heading level0 col1\" >became_member_on</th>        <th class=\"col_heading level0 col2\" >income</th>        <th class=\"col_heading level0 col3\" >missing_data</th>        <th class=\"col_heading level0 col4\" >F</th>        <th class=\"col_heading level0 col5\" >M</th>        <th class=\"col_heading level0 col6\" >O</th>        <th class=\"col_heading level0 col7\" >U</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26level0_row0\" class=\"row_heading level0 row0\" >age</th>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row0_col0\" class=\"data row0 col0\" >1.00</td>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row0_col1\" class=\"data row0 col1\" >0.02</td>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row0_col2\" class=\"data row0 col2\" >0.31</td>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row0_col3\" class=\"data row0 col3\" >0.79</td>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row0_col4\" class=\"data row0 col4\" >0.14</td>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row0_col5\" class=\"data row0 col5\" >0.39</td>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row0_col6\" class=\"data row0 col6\" >0.03</td>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row0_col7\" class=\"data row0 col7\" >0.79</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26level0_row1\" class=\"row_heading level0 row1\" >became_member_on</th>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row1_col0\" class=\"data row1 col0\" >0.02</td>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row1_col1\" class=\"data row1 col1\" >1.00</td>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row1_col2\" class=\"data row1 col2\" >0.03</td>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row1_col3\" class=\"data row1 col3\" >0.03</td>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row1_col4\" class=\"data row1 col4\" >0.01</td>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row1_col5\" class=\"data row1 col5\" >0.03</td>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row1_col6\" class=\"data row1 col6\" >0.01</td>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row1_col7\" class=\"data row1 col7\" >0.03</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26level0_row2\" class=\"row_heading level0 row2\" >income</th>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row2_col0\" class=\"data row2 col0\" >0.31</td>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row2_col1\" class=\"data row2 col1\" >0.03</td>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row2_col2\" class=\"data row2 col2\" >1.00</td>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row2_col3\" class=\"data row2 col3\" >nan</td>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row2_col4\" class=\"data row2 col4\" >0.23</td>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row2_col5\" class=\"data row2 col5\" >0.23</td>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row2_col6\" class=\"data row2 col6\" >0.01</td>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row2_col7\" class=\"data row2 col7\" >nan</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26level0_row3\" class=\"row_heading level0 row3\" >missing_data</th>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row3_col0\" class=\"data row3 col0\" >0.79</td>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row3_col1\" class=\"data row3 col1\" >0.03</td>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row3_col2\" class=\"data row3 col2\" >nan</td>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row3_col3\" class=\"data row3 col3\" >1.00</td>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row3_col4\" class=\"data row3 col4\" >0.29</td>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row3_col5\" class=\"data row3 col5\" >0.38</td>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row3_col6\" class=\"data row3 col6\" >0.04</td>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row3_col7\" class=\"data row3 col7\" >1.00</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26level0_row4\" class=\"row_heading level0 row4\" >F</th>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row4_col0\" class=\"data row4 col0\" >0.14</td>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row4_col1\" class=\"data row4 col1\" >0.01</td>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row4_col2\" class=\"data row4 col2\" >0.23</td>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row4_col3\" class=\"data row4 col3\" >0.29</td>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row4_col4\" class=\"data row4 col4\" >1.00</td>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row4_col5\" class=\"data row4 col5\" >0.75</td>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row4_col6\" class=\"data row4 col6\" >0.08</td>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row4_col7\" class=\"data row4 col7\" >0.29</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26level0_row5\" class=\"row_heading level0 row5\" >M</th>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row5_col0\" class=\"data row5 col0\" >0.39</td>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row5_col1\" class=\"data row5 col1\" >0.03</td>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row5_col2\" class=\"data row5 col2\" >0.23</td>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row5_col3\" class=\"data row5 col3\" >0.38</td>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row5_col4\" class=\"data row5 col4\" >0.75</td>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row5_col5\" class=\"data row5 col5\" >1.00</td>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row5_col6\" class=\"data row5 col6\" >0.11</td>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row5_col7\" class=\"data row5 col7\" >0.38</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26level0_row6\" class=\"row_heading level0 row6\" >O</th>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row6_col0\" class=\"data row6 col0\" >0.03</td>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row6_col1\" class=\"data row6 col1\" >0.01</td>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row6_col2\" class=\"data row6 col2\" >0.01</td>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row6_col3\" class=\"data row6 col3\" >0.04</td>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row6_col4\" class=\"data row6 col4\" >0.08</td>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row6_col5\" class=\"data row6 col5\" >0.11</td>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row6_col6\" class=\"data row6 col6\" >1.00</td>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row6_col7\" class=\"data row6 col7\" >0.04</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26level0_row7\" class=\"row_heading level0 row7\" >U</th>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row7_col0\" class=\"data row7 col0\" >0.79</td>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row7_col1\" class=\"data row7 col1\" >0.03</td>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row7_col2\" class=\"data row7 col2\" >nan</td>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row7_col3\" class=\"data row7 col3\" >1.00</td>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row7_col4\" class=\"data row7 col4\" >0.29</td>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row7_col5\" class=\"data row7 col5\" >0.38</td>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row7_col6\" class=\"data row7 col6\" >0.04</td>\n",
       "                        <td id=\"T_44793256_4b91_11ea_9436_17ddfb4f7b26row7_col7\" class=\"data row7 col7\" >1.00</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7ff710588940>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(pd.DataFrame(profile_df.describe()).round(2).style \\\n",
    "        .set_caption('Dataset description'))\n",
    "\n",
    "display(profile_df.corr().abs().style \\\n",
    "        .set_caption('Pairwise correlation'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Resulting dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>became_member_on</th>\n",
       "      <th>income</th>\n",
       "      <th>missing_data</th>\n",
       "      <th>F</th>\n",
       "      <th>M</th>\n",
       "      <th>O</th>\n",
       "      <th>U</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>68be06ca386d4c31939f3a4f0e3dd783</th>\n",
       "      <td>118</td>\n",
       "      <td>1486857600000000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0610b486422d4921ae7d2bf64640c50b</th>\n",
       "      <td>55</td>\n",
       "      <td>1500076800000000000</td>\n",
       "      <td>112000.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38fe809add3b4fcf9315a9694bb96ff5</th>\n",
       "      <td>118</td>\n",
       "      <td>1531353600000000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78afa995795e4d85b5d9ceeca43f5fef</th>\n",
       "      <td>75</td>\n",
       "      <td>1494288000000000000</td>\n",
       "      <td>100000.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>a03223e636434f42ac4c3df47e8bac43</th>\n",
       "      <td>118</td>\n",
       "      <td>1501804800000000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>e2127556f4f64592b11af22de27a7932</th>\n",
       "      <td>68</td>\n",
       "      <td>1524700800000000000</td>\n",
       "      <td>70000.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8ec6ce2a7e7949b1bf142def7d0e0586</th>\n",
       "      <td>118</td>\n",
       "      <td>1506297600000000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68617ca6246f4fbc85e91a2a49552598</th>\n",
       "      <td>118</td>\n",
       "      <td>1506902400000000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>389bc3fa690240e798340f5a15918d5c</th>\n",
       "      <td>65</td>\n",
       "      <td>1518134400000000000</td>\n",
       "      <td>53000.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8974fc5686fe429db53ddde067b88302</th>\n",
       "      <td>118</td>\n",
       "      <td>1479772800000000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  age     became_member_on    income  \\\n",
       "id                                                                     \n",
       "68be06ca386d4c31939f3a4f0e3dd783  118  1486857600000000000       NaN   \n",
       "0610b486422d4921ae7d2bf64640c50b   55  1500076800000000000  112000.0   \n",
       "38fe809add3b4fcf9315a9694bb96ff5  118  1531353600000000000       NaN   \n",
       "78afa995795e4d85b5d9ceeca43f5fef   75  1494288000000000000  100000.0   \n",
       "a03223e636434f42ac4c3df47e8bac43  118  1501804800000000000       NaN   \n",
       "e2127556f4f64592b11af22de27a7932   68  1524700800000000000   70000.0   \n",
       "8ec6ce2a7e7949b1bf142def7d0e0586  118  1506297600000000000       NaN   \n",
       "68617ca6246f4fbc85e91a2a49552598  118  1506902400000000000       NaN   \n",
       "389bc3fa690240e798340f5a15918d5c   65  1518134400000000000   53000.0   \n",
       "8974fc5686fe429db53ddde067b88302  118  1479772800000000000       NaN   \n",
       "\n",
       "                                  missing_data  F  M  O  U  \n",
       "id                                                          \n",
       "68be06ca386d4c31939f3a4f0e3dd783             1  0  0  0  1  \n",
       "0610b486422d4921ae7d2bf64640c50b             0  1  0  0  0  \n",
       "38fe809add3b4fcf9315a9694bb96ff5             1  0  0  0  1  \n",
       "78afa995795e4d85b5d9ceeca43f5fef             0  1  0  0  0  \n",
       "a03223e636434f42ac4c3df47e8bac43             1  0  0  0  1  \n",
       "e2127556f4f64592b11af22de27a7932             0  0  1  0  0  \n",
       "8ec6ce2a7e7949b1bf142def7d0e0586             1  0  0  0  1  \n",
       "68617ca6246f4fbc85e91a2a49552598             1  0  0  0  1  \n",
       "389bc3fa690240e798340f5a15918d5c             0  0  1  0  0  \n",
       "8974fc5686fe429db53ddde067b88302             1  0  0  0  1  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(profile_df.head(10))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Transcript data set\n",
    "Event log containing records for transactions, offers received, offers viewed, and offers completed"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Overview"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>person</th>\n",
       "      <th>event</th>\n",
       "      <th>value</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>78afa995795e4d85b5d9ceeca43f5fef</td>\n",
       "      <td>offer received</td>\n",
       "      <td>{'offer id': '9b98b8c7a33c4b65b9aebfe6a799e6d9'}</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>a03223e636434f42ac4c3df47e8bac43</td>\n",
       "      <td>offer received</td>\n",
       "      <td>{'offer id': '0b1e1539f2cc45b7b9fa7c272da2e1d7'}</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>e2127556f4f64592b11af22de27a7932</td>\n",
       "      <td>offer received</td>\n",
       "      <td>{'offer id': '2906b810c7d4411798c6938adc9daaa5'}</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8ec6ce2a7e7949b1bf142def7d0e0586</td>\n",
       "      <td>offer received</td>\n",
       "      <td>{'offer id': 'fafdcd668e3743c1bb461111dcafc2a4'}</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>68617ca6246f4fbc85e91a2a49552598</td>\n",
       "      <td>offer received</td>\n",
       "      <td>{'offer id': '4d5c57ea9a6940dd891ad53e9dbe8da0'}</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             person           event  \\\n",
       "0  78afa995795e4d85b5d9ceeca43f5fef  offer received   \n",
       "1  a03223e636434f42ac4c3df47e8bac43  offer received   \n",
       "2  e2127556f4f64592b11af22de27a7932  offer received   \n",
       "3  8ec6ce2a7e7949b1bf142def7d0e0586  offer received   \n",
       "4  68617ca6246f4fbc85e91a2a49552598  offer received   \n",
       "\n",
       "                                              value  time  \n",
       "0  {'offer id': '9b98b8c7a33c4b65b9aebfe6a799e6d9'}     0  \n",
       "1  {'offer id': '0b1e1539f2cc45b7b9fa7c272da2e1d7'}     0  \n",
       "2  {'offer id': '2906b810c7d4411798c6938adc9daaa5'}     0  \n",
       "3  {'offer id': 'fafdcd668e3743c1bb461111dcafc2a4'}     0  \n",
       "4  {'offer id': '4d5c57ea9a6940dd891ad53e9dbe8da0'}     0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "transcript_df = load_dataframe(transcript_data_path)\n",
    "display(transcript_df.head())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Transformation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>person</th>\n",
       "      <th>event</th>\n",
       "      <th>time</th>\n",
       "      <th>amount</th>\n",
       "      <th>offer_id</th>\n",
       "      <th>reward</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>78afa995795e4d85b5d9ceeca43f5fef</td>\n",
       "      <td>offer received</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9b98b8c7a33c4b65b9aebfe6a799e6d9</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>a03223e636434f42ac4c3df47e8bac43</td>\n",
       "      <td>offer received</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0b1e1539f2cc45b7b9fa7c272da2e1d7</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>e2127556f4f64592b11af22de27a7932</td>\n",
       "      <td>offer received</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2906b810c7d4411798c6938adc9daaa5</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8ec6ce2a7e7949b1bf142def7d0e0586</td>\n",
       "      <td>offer received</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>fafdcd668e3743c1bb461111dcafc2a4</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>68617ca6246f4fbc85e91a2a49552598</td>\n",
       "      <td>offer received</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4d5c57ea9a6940dd891ad53e9dbe8da0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>306529</th>\n",
       "      <td>b3a1272bc9904337b331bf348c3e8c17</td>\n",
       "      <td>transaction</td>\n",
       "      <td>714</td>\n",
       "      <td>1.59</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>306530</th>\n",
       "      <td>68213b08d99a4ae1b0dcb72aebd9aa35</td>\n",
       "      <td>transaction</td>\n",
       "      <td>714</td>\n",
       "      <td>9.53</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>306531</th>\n",
       "      <td>a00058cf10334a308c68e7631c529907</td>\n",
       "      <td>transaction</td>\n",
       "      <td>714</td>\n",
       "      <td>3.61</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>306532</th>\n",
       "      <td>76ddbd6576844afe811f1a3c0fbb5bec</td>\n",
       "      <td>transaction</td>\n",
       "      <td>714</td>\n",
       "      <td>3.53</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>306533</th>\n",
       "      <td>c02b10e8752c4d8e9b73f918558531f7</td>\n",
       "      <td>transaction</td>\n",
       "      <td>714</td>\n",
       "      <td>4.05</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>306534 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  person           event  time  amount  \\\n",
       "0       78afa995795e4d85b5d9ceeca43f5fef  offer received     0     NaN   \n",
       "1       a03223e636434f42ac4c3df47e8bac43  offer received     0     NaN   \n",
       "2       e2127556f4f64592b11af22de27a7932  offer received     0     NaN   \n",
       "3       8ec6ce2a7e7949b1bf142def7d0e0586  offer received     0     NaN   \n",
       "4       68617ca6246f4fbc85e91a2a49552598  offer received     0     NaN   \n",
       "...                                  ...             ...   ...     ...   \n",
       "306529  b3a1272bc9904337b331bf348c3e8c17     transaction   714    1.59   \n",
       "306530  68213b08d99a4ae1b0dcb72aebd9aa35     transaction   714    9.53   \n",
       "306531  a00058cf10334a308c68e7631c529907     transaction   714    3.61   \n",
       "306532  76ddbd6576844afe811f1a3c0fbb5bec     transaction   714    3.53   \n",
       "306533  c02b10e8752c4d8e9b73f918558531f7     transaction   714    4.05   \n",
       "\n",
       "                                offer_id  reward  \n",
       "0       9b98b8c7a33c4b65b9aebfe6a799e6d9     NaN  \n",
       "1       0b1e1539f2cc45b7b9fa7c272da2e1d7     NaN  \n",
       "2       2906b810c7d4411798c6938adc9daaa5     NaN  \n",
       "3       fafdcd668e3743c1bb461111dcafc2a4     NaN  \n",
       "4       4d5c57ea9a6940dd891ad53e9dbe8da0     NaN  \n",
       "...                                  ...     ...  \n",
       "306529                               NaN     NaN  \n",
       "306530                               NaN     NaN  \n",
       "306531                               NaN     NaN  \n",
       "306532                               NaN     NaN  \n",
       "306533                               NaN     NaN  \n",
       "\n",
       "[306534 rows x 6 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "transcript_df = transcript_df.join(\n",
    "    pd.DataFrame.from_records(transcript_df.pop('value')))\n",
    "transcript_df.offer_id.update(transcript_df.pop('offer id'))\n",
    "\n",
    "display(transcript_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "users_to_discard = np.append(\n",
    "    transcript_df.query('amount >= 50').person.unique(),\n",
    "    transcript_df.groupby('person').count().query('amount == 0').index.values)\n",
    "\n",
    "profile_df = profile_df.query('id not in @users_to_discard')\n",
    "transcript_df = transcript_df.query('person not in @users_to_discard')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Transformation after data cleaning\n",
    "After removing some registers from the profile datase, it is time to scale features and fill blank values."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = profile_df.filter(['became_member_on'])\n",
    "data_transf = quantile_transform(data, output_distribution='normal', copy=True)\n",
    "data = pd.DataFrame(data_transf, columns=data.columns, index=data.index)\n",
    "profile_df.update(data)\n",
    "\n",
    "data = profile_df.query('missing_data == 0').filter(['age','income'])\n",
    "# data_transf = quantile_transform(data, output_distribution='normal', copy=False)\n",
    "data_transf = robust_scale(data, copy=False)\n",
    "data = pd.DataFrame(data_transf, columns=data.columns, index=data.index)\n",
    "profile_df.update(data)\n",
    "\n",
    "missing_data = profile_df.income.isna()\n",
    "income_mean = profile_df[~(missing_data.values)].income.mean()\n",
    "age_mean = profile_df[~(missing_data.values)].age.mean()\n",
    "\n",
    "# # # fill age and income with its respective mean values\n",
    "profile_df.loc[missing_data.values, 'age'] = age_mean\n",
    "profile_df.loc[missing_data.values, 'income'] = income_mean"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Classify offer sendings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Split register according to event\n",
    "offer_received_df = transcript_df.query('event == \"offer received\"')\n",
    "offer_viewed_df = transcript_df.query('event == \"offer viewed\"')\n",
    "offer_completed_df = transcript_df.query('event == \"offer completed\"')\n",
    "transaction_df = transcript_df.query('event == \"transaction\"')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "## Remove unnecessary columns\n",
    "offer_received_df = offer_received_df.drop(columns=['amount', 'reward'])\n",
    "\n",
    "## Create registers when customers did not receive offer\n",
    "offer_sending_time = offer_received_df.time.unique()\n",
    "new_index = pd.MultiIndex.from_product(\n",
    "    (profile_df.index, offer_sending_time),\n",
    "    names=['person', 'time'])\n",
    "\n",
    "offer_received_df = offer_received_df.set_index(['person', 'time'])\n",
    "offer_received_df = offer_received_df.sort_index()\n",
    "offer_received_df = offer_received_df.reindex(new_index)\n",
    "offer_received_df = offer_received_df.reset_index()\n",
    "\n",
    "## Create column to indicate when the offer ends\n",
    "offer_received_df = offer_received_df.join(portfolio_df.duration*24, on='offer_id')\n",
    "offer_received_df = offer_received_df.assign(\n",
    "    offer_ends_on=offer_received_df.time + offer_received_df.duration)\n",
    "\n",
    "## Create column to indicate when the offer is informational\n",
    "offer_received_df = offer_received_df.join(\n",
    "    portfolio_df.informational, on='offer_id')\n",
    "\n",
    "## Create column to indicate when the offer is viewed or completed\n",
    "offer_received_df = offer_received_df.assign(viewed_on=np.nan, completed_on=np.nan)\n",
    "\n",
    "## Create a column to hold the label for that offer sending\n",
    "offer_received_df = offer_received_df.assign(label=np.nan)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 21min 48s, sys: 294 ms, total: 21min 48s\n",
      "Wall time: 21min 49s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "next_sending_time = {0: 168, 168: 336, 336: 408,\n",
    "                     408: 504, 504: 576, 576: 720}\n",
    "\n",
    "def classify_offers(row):\n",
    "    if row.offer_id is np.nan:\n",
    "        row.offer_ends_on = next_sending_time[row.time]\n",
    "        try:\n",
    "            # In this case, there is no offer to be viewed or completed.\n",
    "            # So we look for a simple transaction.\n",
    "            row.completed_on = transaction_df.query(\n",
    "                'person == @row.person ' \\\n",
    "                'and time >= @row.time ' \\\n",
    "                'and time <= @row.offer_ends_on').time.values[0]\n",
    "        except:\n",
    "            # If there is no transaction in this period\n",
    "            row.label = 0\n",
    "        else:\n",
    "            row.label = 1\n",
    "        finally:\n",
    "            return row\n",
    "\n",
    "    try:\n",
    "        row.viewed_on = offer_viewed_df.query(\n",
    "            'person == @row.person ' \\\n",
    "            'and offer_id == @row.offer_id ' \\\n",
    "            'and time >= @row.time ' \\\n",
    "            'and time <= @row.offer_ends_on').time.values[0]\n",
    "    except:\n",
    "        # Offer was not viewed\n",
    "        row.label = 0\n",
    "        return row\n",
    "\n",
    "    \n",
    "    if row.informational == 1:\n",
    "        try:\n",
    "            # In this case, there is no offer to be completed.\n",
    "            # So we look for a simple transaction after offer viewed\n",
    "            row.completed_on = transaction_df.query(\n",
    "                'person == @row.person ' \\\n",
    "                'and time >= @row.viewed_on ' \\\n",
    "                'and time <= @row.offer_ends_on').time.values[0]\n",
    "        except:\n",
    "            # If there is no transaction in this period\n",
    "            row.label = 0\n",
    "        else:\n",
    "            row.label = 1\n",
    "        finally:\n",
    "            return row\n",
    "\n",
    "\n",
    "    try:\n",
    "        # In the other cases, we need an offer completion\n",
    "        row.completed_on = offer_completed_df.query(\n",
    "            'person == @row.person ' \\\n",
    "            'and offer_id == @row.offer_id ' \\\n",
    "            'and time >= @row.viewed_on ' \\\n",
    "            'and time <= @row.offer_ends_on').time.values[0]\n",
    "    except:\n",
    "        # If the offer was not completed\n",
    "        row.label = 0\n",
    "    else:\n",
    "        row.label = 1\n",
    "    finally:\n",
    "        return row\n",
    "\n",
    "\n",
    "offer_received_df = offer_received_df.apply(classify_offers, axis=1)\n",
    "\n",
    "## Remove auxiliary columns\n",
    "offer_received_df.drop(\n",
    "    inplace=True,\n",
    "    columns=['duration', 'offer_ends_on', 'informational',\n",
    "             'viewed_on', 'completed_on'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Transformation after classifying offers\n",
    "After removing some registers from the profile datase, it is time to scale features and fill blank values."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>reward</th>\n",
       "      <th>difficulty</th>\n",
       "      <th>duration</th>\n",
       "      <th>bogo</th>\n",
       "      <th>discount</th>\n",
       "      <th>informational</th>\n",
       "      <th>mobile</th>\n",
       "      <th>social</th>\n",
       "      <th>web</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ae264e3637204a6fb9bb56bc8210ddfd</th>\n",
       "      <td>1.71</td>\n",
       "      <td>0.42</td>\n",
       "      <td>0.23</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4d5c57ea9a6940dd891ad53e9dbe8da0</th>\n",
       "      <td>1.71</td>\n",
       "      <td>0.42</td>\n",
       "      <td>-0.68</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3f207df678b143eea3cee63160fa8bed</th>\n",
       "      <td>-1.24</td>\n",
       "      <td>-1.39</td>\n",
       "      <td>-1.14</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9b98b8c7a33c4b65b9aebfe6a799e6d9</th>\n",
       "      <td>0.24</td>\n",
       "      <td>-0.49</td>\n",
       "      <td>0.23</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0b1e1539f2cc45b7b9fa7c272da2e1d7</th>\n",
       "      <td>0.24</td>\n",
       "      <td>2.22</td>\n",
       "      <td>1.59</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2298d6c36e964ae4a3e7e9706d1fb8c2</th>\n",
       "      <td>-0.35</td>\n",
       "      <td>-0.13</td>\n",
       "      <td>0.23</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fafdcd668e3743c1bb461111dcafc2a4</th>\n",
       "      <td>-0.65</td>\n",
       "      <td>0.42</td>\n",
       "      <td>1.59</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5a8bc65990b245e5a138643cd4eb9837</th>\n",
       "      <td>-1.24</td>\n",
       "      <td>-1.39</td>\n",
       "      <td>-1.59</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>f19421c1d4aa40978ebb69ca19b0e20d</th>\n",
       "      <td>0.24</td>\n",
       "      <td>-0.49</td>\n",
       "      <td>-0.68</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2906b810c7d4411798c6938adc9daaa5</th>\n",
       "      <td>-0.65</td>\n",
       "      <td>0.42</td>\n",
       "      <td>0.23</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  reward  difficulty  duration  bogo  \\\n",
       "id                                                                     \n",
       "ae264e3637204a6fb9bb56bc8210ddfd    1.71        0.42      0.23     1   \n",
       "4d5c57ea9a6940dd891ad53e9dbe8da0    1.71        0.42     -0.68     1   \n",
       "3f207df678b143eea3cee63160fa8bed   -1.24       -1.39     -1.14     0   \n",
       "9b98b8c7a33c4b65b9aebfe6a799e6d9    0.24       -0.49      0.23     1   \n",
       "0b1e1539f2cc45b7b9fa7c272da2e1d7    0.24        2.22      1.59     0   \n",
       "2298d6c36e964ae4a3e7e9706d1fb8c2   -0.35       -0.13      0.23     0   \n",
       "fafdcd668e3743c1bb461111dcafc2a4   -0.65        0.42      1.59     0   \n",
       "5a8bc65990b245e5a138643cd4eb9837   -1.24       -1.39     -1.59     0   \n",
       "f19421c1d4aa40978ebb69ca19b0e20d    0.24       -0.49     -0.68     1   \n",
       "2906b810c7d4411798c6938adc9daaa5   -0.65        0.42      0.23     0   \n",
       "\n",
       "                                  discount  informational  mobile  social  web  \n",
       "id                                                                              \n",
       "ae264e3637204a6fb9bb56bc8210ddfd         0              0     1.0     1.0  0.0  \n",
       "4d5c57ea9a6940dd891ad53e9dbe8da0         0              0     1.0     1.0  1.0  \n",
       "3f207df678b143eea3cee63160fa8bed         0              1     1.0     0.0  1.0  \n",
       "9b98b8c7a33c4b65b9aebfe6a799e6d9         0              0     1.0     0.0  1.0  \n",
       "0b1e1539f2cc45b7b9fa7c272da2e1d7         1              0     0.0     0.0  1.0  \n",
       "2298d6c36e964ae4a3e7e9706d1fb8c2         1              0     1.0     1.0  1.0  \n",
       "fafdcd668e3743c1bb461111dcafc2a4         1              0     1.0     1.0  1.0  \n",
       "5a8bc65990b245e5a138643cd4eb9837         0              1     1.0     1.0  0.0  \n",
       "f19421c1d4aa40978ebb69ca19b0e20d         0              0     1.0     1.0  1.0  \n",
       "2906b810c7d4411798c6938adc9daaa5         1              0     1.0     0.0  1.0  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Transform PORTFOLIO\n",
    "data = portfolio_df[['reward','difficulty','duration']]\n",
    "data = scale(data)\n",
    "portfolio_df[['reward','difficulty','duration']] = data\n",
    "portfolio_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create Features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Join datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>event</th>\n",
       "      <th>offer_id</th>\n",
       "      <th>label</th>\n",
       "      <th>reward</th>\n",
       "      <th>difficulty</th>\n",
       "      <th>duration</th>\n",
       "      <th>bogo</th>\n",
       "      <th>discount</th>\n",
       "      <th>informational</th>\n",
       "      <th>mobile</th>\n",
       "      <th>social</th>\n",
       "      <th>web</th>\n",
       "      <th>age</th>\n",
       "      <th>became_member_on</th>\n",
       "      <th>income</th>\n",
       "      <th>missing_data</th>\n",
       "      <th>F</th>\n",
       "      <th>M</th>\n",
       "      <th>O</th>\n",
       "      <th>U</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>person</th>\n",
       "      <th>time</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">0009655768c64bdeb2e877511632db8f</th>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.92</td>\n",
       "      <td>-0.15</td>\n",
       "      <td>0.30</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>168</th>\n",
       "      <td>offer received</td>\n",
       "      <td>5a8bc65990b245e5a138643cd4eb9837</td>\n",
       "      <td>1</td>\n",
       "      <td>-1.24</td>\n",
       "      <td>-1.39</td>\n",
       "      <td>-1.59</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.92</td>\n",
       "      <td>-0.15</td>\n",
       "      <td>0.30</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>336</th>\n",
       "      <td>offer received</td>\n",
       "      <td>3f207df678b143eea3cee63160fa8bed</td>\n",
       "      <td>1</td>\n",
       "      <td>-1.24</td>\n",
       "      <td>-1.39</td>\n",
       "      <td>-1.14</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-0.92</td>\n",
       "      <td>-0.15</td>\n",
       "      <td>0.30</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>408</th>\n",
       "      <td>offer received</td>\n",
       "      <td>f19421c1d4aa40978ebb69ca19b0e20d</td>\n",
       "      <td>0</td>\n",
       "      <td>0.24</td>\n",
       "      <td>-0.49</td>\n",
       "      <td>-0.68</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-0.92</td>\n",
       "      <td>-0.15</td>\n",
       "      <td>0.30</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>504</th>\n",
       "      <td>offer received</td>\n",
       "      <td>fafdcd668e3743c1bb461111dcafc2a4</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.65</td>\n",
       "      <td>0.42</td>\n",
       "      <td>1.59</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-0.92</td>\n",
       "      <td>-0.15</td>\n",
       "      <td>0.30</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">ffff82501cea40309d5fdd7edcca4a07</th>\n",
       "      <th>168</th>\n",
       "      <td>offer received</td>\n",
       "      <td>0b1e1539f2cc45b7b9fa7c272da2e1d7</td>\n",
       "      <td>1</td>\n",
       "      <td>0.24</td>\n",
       "      <td>2.22</td>\n",
       "      <td>1.59</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-0.42</td>\n",
       "      <td>-0.37</td>\n",
       "      <td>-0.03</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>336</th>\n",
       "      <td>offer received</td>\n",
       "      <td>2906b810c7d4411798c6938adc9daaa5</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.65</td>\n",
       "      <td>0.42</td>\n",
       "      <td>0.23</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-0.42</td>\n",
       "      <td>-0.37</td>\n",
       "      <td>-0.03</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>408</th>\n",
       "      <td>offer received</td>\n",
       "      <td>2906b810c7d4411798c6938adc9daaa5</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.65</td>\n",
       "      <td>0.42</td>\n",
       "      <td>0.23</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-0.42</td>\n",
       "      <td>-0.37</td>\n",
       "      <td>-0.03</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>504</th>\n",
       "      <td>offer received</td>\n",
       "      <td>9b98b8c7a33c4b65b9aebfe6a799e6d9</td>\n",
       "      <td>0</td>\n",
       "      <td>0.24</td>\n",
       "      <td>-0.49</td>\n",
       "      <td>0.23</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-0.42</td>\n",
       "      <td>-0.37</td>\n",
       "      <td>-0.03</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>576</th>\n",
       "      <td>offer received</td>\n",
       "      <td>2906b810c7d4411798c6938adc9daaa5</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.65</td>\n",
       "      <td>0.42</td>\n",
       "      <td>0.23</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-0.42</td>\n",
       "      <td>-0.37</td>\n",
       "      <td>-0.03</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>95346 rows × 20 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                event  \\\n",
       "person                           time                   \n",
       "0009655768c64bdeb2e877511632db8f 0                  0   \n",
       "                                 168   offer received   \n",
       "                                 336   offer received   \n",
       "                                 408   offer received   \n",
       "                                 504   offer received   \n",
       "...                                               ...   \n",
       "ffff82501cea40309d5fdd7edcca4a07 168   offer received   \n",
       "                                 336   offer received   \n",
       "                                 408   offer received   \n",
       "                                 504   offer received   \n",
       "                                 576   offer received   \n",
       "\n",
       "                                                               offer_id  \\\n",
       "person                           time                                     \n",
       "0009655768c64bdeb2e877511632db8f 0                                    0   \n",
       "                                 168   5a8bc65990b245e5a138643cd4eb9837   \n",
       "                                 336   3f207df678b143eea3cee63160fa8bed   \n",
       "                                 408   f19421c1d4aa40978ebb69ca19b0e20d   \n",
       "                                 504   fafdcd668e3743c1bb461111dcafc2a4   \n",
       "...                                                                 ...   \n",
       "ffff82501cea40309d5fdd7edcca4a07 168   0b1e1539f2cc45b7b9fa7c272da2e1d7   \n",
       "                                 336   2906b810c7d4411798c6938adc9daaa5   \n",
       "                                 408   2906b810c7d4411798c6938adc9daaa5   \n",
       "                                 504   9b98b8c7a33c4b65b9aebfe6a799e6d9   \n",
       "                                 576   2906b810c7d4411798c6938adc9daaa5   \n",
       "\n",
       "                                       label  reward  difficulty  duration  \\\n",
       "person                           time                                        \n",
       "0009655768c64bdeb2e877511632db8f 0         0    0.00        0.00      0.00   \n",
       "                                 168       1   -1.24       -1.39     -1.59   \n",
       "                                 336       1   -1.24       -1.39     -1.14   \n",
       "                                 408       0    0.24       -0.49     -0.68   \n",
       "                                 504       0   -0.65        0.42      1.59   \n",
       "...                                      ...     ...         ...       ...   \n",
       "ffff82501cea40309d5fdd7edcca4a07 168       1    0.24        2.22      1.59   \n",
       "                                 336       1   -0.65        0.42      0.23   \n",
       "                                 408       1   -0.65        0.42      0.23   \n",
       "                                 504       0    0.24       -0.49      0.23   \n",
       "                                 576       0   -0.65        0.42      0.23   \n",
       "\n",
       "                                       bogo  discount  informational  mobile  \\\n",
       "person                           time                                          \n",
       "0009655768c64bdeb2e877511632db8f 0      0.0       0.0            0.0     0.0   \n",
       "                                 168    0.0       0.0            1.0     1.0   \n",
       "                                 336    0.0       0.0            1.0     1.0   \n",
       "                                 408    1.0       0.0            0.0     1.0   \n",
       "                                 504    0.0       1.0            0.0     1.0   \n",
       "...                                     ...       ...            ...     ...   \n",
       "ffff82501cea40309d5fdd7edcca4a07 168    0.0       1.0            0.0     0.0   \n",
       "                                 336    0.0       1.0            0.0     1.0   \n",
       "                                 408    0.0       1.0            0.0     1.0   \n",
       "                                 504    1.0       0.0            0.0     1.0   \n",
       "                                 576    0.0       1.0            0.0     1.0   \n",
       "\n",
       "                                       social  web   age  became_member_on  \\\n",
       "person                           time                                        \n",
       "0009655768c64bdeb2e877511632db8f 0        0.0  0.0 -0.92             -0.15   \n",
       "                                 168      1.0  0.0 -0.92             -0.15   \n",
       "                                 336      0.0  1.0 -0.92             -0.15   \n",
       "                                 408      1.0  1.0 -0.92             -0.15   \n",
       "                                 504      1.0  1.0 -0.92             -0.15   \n",
       "...                                       ...  ...   ...               ...   \n",
       "ffff82501cea40309d5fdd7edcca4a07 168      0.0  1.0 -0.42             -0.37   \n",
       "                                 336      0.0  1.0 -0.42             -0.37   \n",
       "                                 408      0.0  1.0 -0.42             -0.37   \n",
       "                                 504      0.0  1.0 -0.42             -0.37   \n",
       "                                 576      0.0  1.0 -0.42             -0.37   \n",
       "\n",
       "                                       income  missing_data  F  M  O  U  \n",
       "person                           time                                    \n",
       "0009655768c64bdeb2e877511632db8f 0       0.30             0  0  1  0  0  \n",
       "                                 168     0.30             0  0  1  0  0  \n",
       "                                 336     0.30             0  0  1  0  0  \n",
       "                                 408     0.30             0  0  1  0  0  \n",
       "                                 504     0.30             0  0  1  0  0  \n",
       "...                                       ...           ... .. .. .. ..  \n",
       "ffff82501cea40309d5fdd7edcca4a07 168    -0.03             0  1  0  0  0  \n",
       "                                 336    -0.03             0  1  0  0  0  \n",
       "                                 408    -0.03             0  1  0  0  0  \n",
       "                                 504    -0.03             0  1  0  0  0  \n",
       "                                 576    -0.03             0  1  0  0  0  \n",
       "\n",
       "[95346 rows x 20 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "## Set index as person and time\n",
    "offer_received_df = offer_received_df.set_index(['person', 'time'])\n",
    "offer_received_df = offer_received_df.sort_index()\n",
    "\n",
    "## Join offer data with portfolio and profile data\n",
    "offer_received_df = offer_received_df.join(portfolio_df, on='offer_id')\n",
    "offer_received_df = offer_received_df.join(profile_df, on='person')\n",
    "\n",
    "## Fill NA\n",
    "offer_received_df.fillna(value=0, inplace=True)\n",
    "\n",
    "display(offer_received_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Plot the correlation matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>reward</th>\n",
       "      <th>difficulty</th>\n",
       "      <th>duration</th>\n",
       "      <th>bogo</th>\n",
       "      <th>discount</th>\n",
       "      <th>informational</th>\n",
       "      <th>mobile</th>\n",
       "      <th>social</th>\n",
       "      <th>web</th>\n",
       "      <th>age</th>\n",
       "      <th>became_member_on</th>\n",
       "      <th>income</th>\n",
       "      <th>missing_data</th>\n",
       "      <th>F</th>\n",
       "      <th>M</th>\n",
       "      <th>O</th>\n",
       "      <th>U</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>label</th>\n",
       "      <td>1.00</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.08</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.10</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.08</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.14</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.15</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.13</td>\n",
       "      <td>0.08</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>reward</th>\n",
       "      <td>0.06</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.47</td>\n",
       "      <td>0.16</td>\n",
       "      <td>0.73</td>\n",
       "      <td>0.26</td>\n",
       "      <td>0.60</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.24</td>\n",
       "      <td>0.08</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>difficulty</th>\n",
       "      <td>0.08</td>\n",
       "      <td>0.47</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.81</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.55</td>\n",
       "      <td>0.67</td>\n",
       "      <td>0.41</td>\n",
       "      <td>0.13</td>\n",
       "      <td>0.17</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>duration</th>\n",
       "      <td>0.01</td>\n",
       "      <td>0.16</td>\n",
       "      <td>0.81</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.17</td>\n",
       "      <td>0.69</td>\n",
       "      <td>0.66</td>\n",
       "      <td>0.29</td>\n",
       "      <td>0.16</td>\n",
       "      <td>0.24</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bogo</th>\n",
       "      <td>0.10</td>\n",
       "      <td>0.73</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.17</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.43</td>\n",
       "      <td>0.27</td>\n",
       "      <td>0.46</td>\n",
       "      <td>0.39</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>discount</th>\n",
       "      <td>0.03</td>\n",
       "      <td>0.26</td>\n",
       "      <td>0.55</td>\n",
       "      <td>0.69</td>\n",
       "      <td>0.43</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.27</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.07</td>\n",
       "      <td>0.54</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>informational</th>\n",
       "      <td>0.04</td>\n",
       "      <td>0.60</td>\n",
       "      <td>0.67</td>\n",
       "      <td>0.66</td>\n",
       "      <td>0.27</td>\n",
       "      <td>0.27</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.29</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.08</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mobile</th>\n",
       "      <td>0.08</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.41</td>\n",
       "      <td>0.29</td>\n",
       "      <td>0.46</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.29</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.63</td>\n",
       "      <td>0.52</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>social</th>\n",
       "      <td>0.06</td>\n",
       "      <td>0.24</td>\n",
       "      <td>0.13</td>\n",
       "      <td>0.16</td>\n",
       "      <td>0.39</td>\n",
       "      <td>0.07</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.63</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.12</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>web</th>\n",
       "      <td>0.14</td>\n",
       "      <td>0.08</td>\n",
       "      <td>0.17</td>\n",
       "      <td>0.24</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.54</td>\n",
       "      <td>0.08</td>\n",
       "      <td>0.52</td>\n",
       "      <td>0.12</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>age</th>\n",
       "      <td>0.03</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.30</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.15</td>\n",
       "      <td>0.14</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>became_member_on</th>\n",
       "      <td>0.15</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.03</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>income</th>\n",
       "      <td>0.05</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.30</td>\n",
       "      <td>0.06</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.22</td>\n",
       "      <td>0.21</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>missing_data</th>\n",
       "      <td>0.13</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.29</td>\n",
       "      <td>0.38</td>\n",
       "      <td>0.04</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>F</th>\n",
       "      <td>0.08</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.15</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.22</td>\n",
       "      <td>0.29</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.75</td>\n",
       "      <td>0.08</td>\n",
       "      <td>0.29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>M</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.14</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.21</td>\n",
       "      <td>0.38</td>\n",
       "      <td>0.75</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>O</th>\n",
       "      <td>0.02</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.08</td>\n",
       "      <td>0.11</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>U</th>\n",
       "      <td>0.13</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.29</td>\n",
       "      <td>0.38</td>\n",
       "      <td>0.04</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  label  reward  difficulty  duration  bogo  discount  \\\n",
       "label              1.00    0.06        0.08      0.01  0.10      0.03   \n",
       "reward             0.06    1.00        0.47      0.16  0.73      0.26   \n",
       "difficulty         0.08    0.47        1.00      0.81  0.03      0.55   \n",
       "duration           0.01    0.16        0.81      1.00  0.17      0.69   \n",
       "bogo               0.10    0.73        0.03      0.17  1.00      0.43   \n",
       "discount           0.03    0.26        0.55      0.69  0.43      1.00   \n",
       "informational      0.04    0.60        0.67      0.66  0.27      0.27   \n",
       "mobile             0.08    0.04        0.41      0.29  0.46      0.11   \n",
       "social             0.06    0.24        0.13      0.16  0.39      0.07   \n",
       "web                0.14    0.08        0.17      0.24  0.20      0.54   \n",
       "age                0.03    0.00        0.00      0.00  0.00      0.00   \n",
       "became_member_on   0.15    0.00        0.00      0.00  0.00      0.00   \n",
       "income             0.05    0.00        0.00      0.00  0.00      0.00   \n",
       "missing_data       0.13    0.00        0.00      0.00  0.00      0.00   \n",
       "F                  0.08    0.00        0.00      0.00  0.00      0.00   \n",
       "M                  0.00    0.00        0.00      0.00  0.00      0.01   \n",
       "O                  0.02    0.00        0.00      0.00  0.00      0.00   \n",
       "U                  0.13    0.00        0.00      0.00  0.00      0.00   \n",
       "\n",
       "                  informational  mobile  social   web   age  became_member_on  \\\n",
       "label                      0.04    0.08    0.06  0.14  0.03              0.15   \n",
       "reward                     0.60    0.04    0.24  0.08  0.00              0.00   \n",
       "difficulty                 0.67    0.41    0.13  0.17  0.00              0.00   \n",
       "duration                   0.66    0.29    0.16  0.24  0.00              0.00   \n",
       "bogo                       0.27    0.46    0.39  0.20  0.00              0.00   \n",
       "discount                   0.27    0.11    0.07  0.54  0.00              0.00   \n",
       "informational              1.00    0.29    0.04  0.08  0.00              0.00   \n",
       "mobile                     0.29    1.00    0.63  0.52  0.00              0.00   \n",
       "social                     0.04    0.63    1.00  0.12  0.00              0.01   \n",
       "web                        0.08    0.52    0.12  1.00  0.00              0.00   \n",
       "age                        0.00    0.00    0.00  0.00  1.00              0.03   \n",
       "became_member_on           0.00    0.00    0.01  0.00  0.03              1.00   \n",
       "income                     0.00    0.00    0.00  0.00  0.30              0.06   \n",
       "missing_data               0.00    0.00    0.00  0.00  0.00              0.01   \n",
       "F                          0.00    0.00    0.00  0.00  0.15              0.02   \n",
       "M                          0.00    0.00    0.00  0.00  0.14              0.00   \n",
       "O                          0.00    0.01    0.01  0.01  0.00              0.00   \n",
       "U                          0.00    0.00    0.00  0.00  0.00              0.01   \n",
       "\n",
       "                  income  missing_data     F     M     O     U  \n",
       "label               0.05          0.13  0.08  0.00  0.02  0.13  \n",
       "reward              0.00          0.00  0.00  0.00  0.00  0.00  \n",
       "difficulty          0.00          0.00  0.00  0.00  0.00  0.00  \n",
       "duration            0.00          0.00  0.00  0.00  0.00  0.00  \n",
       "bogo                0.00          0.00  0.00  0.00  0.00  0.00  \n",
       "discount            0.00          0.00  0.00  0.01  0.00  0.00  \n",
       "informational       0.00          0.00  0.00  0.00  0.00  0.00  \n",
       "mobile              0.00          0.00  0.00  0.00  0.01  0.00  \n",
       "social              0.00          0.00  0.00  0.00  0.01  0.00  \n",
       "web                 0.00          0.00  0.00  0.00  0.01  0.00  \n",
       "age                 0.30          0.00  0.15  0.14  0.00  0.00  \n",
       "became_member_on    0.06          0.01  0.02  0.00  0.00  0.01  \n",
       "income              1.00          0.00  0.22  0.21  0.01  0.00  \n",
       "missing_data        0.00          1.00  0.29  0.38  0.04  1.00  \n",
       "F                   0.22          0.29  1.00  0.75  0.08  0.29  \n",
       "M                   0.21          0.38  0.75  1.00  0.11  0.38  \n",
       "O                   0.01          0.04  0.08  0.11  1.00  0.04  \n",
       "U                   0.00          1.00  0.29  0.38  0.04  1.00  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "offer_received_df.corr().abs().round(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here, we see features not correlated with each other, except for the gender *unknown* and *missing_data* in profile dataset. Thus, *missing_data* will be employed whereas *unknown* not."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Organize customer data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 16.3 s, sys: 20 ms, total: 16.4 s\n",
      "Wall time: 16 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "## Certify that dataframe is ordered\n",
    "offer_received_df = offer_received_df.sort_index()\n",
    "\n",
    "## Choose features to feed the networks\n",
    "features_cols = [\n",
    "    'reward', 'difficulty', 'duration',                 # offer characteristics\n",
    "    'bogo', 'discount', 'informational',                # offer type\n",
    "    'mobile', 'social', 'web',                          # channels\n",
    "    'age','became_member_on', 'income', 'missing_data', # customer data\n",
    "    'F', 'M', 'O'                                       # customer gender\n",
    "]\n",
    "\n",
    "## Create feature and target lists where each position\n",
    "# holds the data related to one customer\n",
    "features = []\n",
    "targets = []\n",
    "for index, user_data in offer_received_df.groupby('person'):\n",
    "    targets.append(user_data['label'].values)\n",
    "    features.append(user_data[features_cols].values)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create the data loaders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Convert features and targets to tensors\n",
    "features = torch.as_tensor(features, dtype=torch.float)\n",
    "targets = torch.as_tensor(targets, dtype=torch.long)\n",
    "\n",
    "\n",
    "## Split data into three random datasets \n",
    "\n",
    "# Generate randomic indices\n",
    "len_dataset = len(features)\n",
    "random_idx = np.random.choice(len_dataset, len_dataset, replace=False)\n",
    "\n",
    "# Use the proportions: train: 80%, valid: 10%, test: 10%\n",
    "train_idx = random_idx[:int(len_dataset*0.8)]\n",
    "valid_idx = random_idx[int(len_dataset*0.8):-int(len_dataset*0.1)]\n",
    "test_idx = random_idx[-int(len_dataset*0.1):]\n",
    "\n",
    "# Create datasets\n",
    "train_dataset = TensorDataset(features[train_idx], targets[train_idx])\n",
    "valid_dataset = TensorDataset(features[valid_idx], targets[valid_idx])\n",
    "test_dataset = TensorDataset(features[test_idx], targets[test_idx])\n",
    "\n",
    "# Create dataloaders\n",
    "train_dataloader = DataLoader(train_dataset, batch_size=32, shuffle=True)\n",
    "valid_dataloader = DataLoader(valid_dataset, batch_size=32, shuffle=True)\n",
    "test_dataloader = DataLoader(test_dataset, batch_size=32, shuffle=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Count targets within each dataloader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train dataset:\tCN: 42200 / 55.33%\tCP: 34072 / 44.67%\n",
      "Valid dataset:\tCN:  5255 / 55.08%\tCP:  4285 / 44.92%\n",
      "Test dataset:\tCN:  5247 / 55.03%\tCP:  4287 / 44.97%\n"
     ]
    }
   ],
   "source": [
    "# TARGETS FOR TRAINING\n",
    "targ_neg = (targets[train_idx] == 0).sum()\n",
    "targ_pos = (targets[train_idx] == 1).sum()\n",
    "targ_total = targ_neg + targ_pos\n",
    "print('Train dataset:\\tCN: {:5d} / {:5.2f}%\\tCP: {:5d} / {:5.2f}%' \\\n",
    "      .format(targ_neg, targ_neg*100./targ_total,\n",
    "              targ_pos, targ_pos*100./targ_total))\n",
    "\n",
    "# TARGETS FOR TRAINING\n",
    "targ_neg = (targets[valid_idx] == 0).sum()\n",
    "targ_pos = (targets[valid_idx] == 1).sum()\n",
    "targ_total = targ_neg + targ_pos\n",
    "print('Valid dataset:\\tCN: {:5d} / {:5.2f}%\\tCP: {:5d} / {:5.2f}%' \\\n",
    "      .format(targ_neg, targ_neg*100./targ_total,\n",
    "              targ_pos, targ_pos*100./targ_total))\n",
    "\n",
    "# TARGETS FOR TRAINING\n",
    "targ_neg = (targets[test_idx] == 0).sum()\n",
    "targ_pos = (targets[test_idx] == 1).sum()\n",
    "targ_total = targ_neg + targ_pos\n",
    "print('Test dataset:\\tCN: {:5d} / {:5.2f}%\\tCP: {:5d} / {:5.2f}%' \\\n",
    "      .format(targ_neg, targ_neg*100./targ_total,\n",
    "              targ_pos, targ_pos*100./targ_total))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save the dataloaders into a zip file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  adding: dataloaders.pt (deflated 93%)\n"
     ]
    }
   ],
   "source": [
    "torch.save((train_dataloader, valid_dataloader, test_dataloader), 'dataloaders.pt')\n",
    "!zip dataloaders.zip dataloaders.pt\n",
    "!rm dataloaders.pt"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}