Healthcare for all 2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "643d5422",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "fde1e5a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sqlalchemy import create_engine\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import getpass\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from scipy.stats import chi2_contingency\n",
    "import datetime\n",
    "import warnings\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "from scipy.stats import chi2_contingency\n",
    "\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.decomposition import PCA\n",
    "from imblearn.over_sampling import SMOTE\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.model_selection import train_test_split\n",
    "from imblearn.under_sampling import TomekLinks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "2b755eb0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: xgboost in /Users/genivabraaf/opt/anaconda3/lib/python3.8/site-packages (1.5.2)\n",
      "Requirement already satisfied: numpy in /Users/genivabraaf/opt/anaconda3/lib/python3.8/site-packages (from xgboost) (1.20.1)\n",
      "Requirement already satisfied: scipy in /Users/genivabraaf/opt/anaconda3/lib/python3.8/site-packages (from xgboost) (1.6.2)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install xgboost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "481bf9d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bff37469",
   "metadata": {},
   "source": [
    "## The business question: 1. Are people going to donate or not?"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d8eae322",
   "metadata": {},
   "source": [
    "Exploring the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "4a1d1d41",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/genivabraaf/opt/anaconda3/lib/python3.8/site-packages/IPython/core/interactiveshell.py:3165: DtypeWarning: Columns (8) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  has_raised = await self.run_ast_nodes(code_ast.body, cell_name,\n"
     ]
    }
   ],
   "source": [
    "file = pd.read_csv('learningSet (1).csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "818b0390",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ODATEDW</th>\n",
       "      <th>OSOURCE</th>\n",
       "      <th>TCODE</th>\n",
       "      <th>STATE</th>\n",
       "      <th>ZIP</th>\n",
       "      <th>MAILCODE</th>\n",
       "      <th>PVASTATE</th>\n",
       "      <th>DOB</th>\n",
       "      <th>NOEXCH</th>\n",
       "      <th>RECINHSE</th>\n",
       "      <th>...</th>\n",
       "      <th>TARGET_D</th>\n",
       "      <th>HPHONE_D</th>\n",
       "      <th>RFA_2R</th>\n",
       "      <th>RFA_2F</th>\n",
       "      <th>RFA_2A</th>\n",
       "      <th>MDMAUD_R</th>\n",
       "      <th>MDMAUD_F</th>\n",
       "      <th>MDMAUD_A</th>\n",
       "      <th>CLUSTER2</th>\n",
       "      <th>GEOCODE2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8901</td>\n",
       "      <td>GRI</td>\n",
       "      <td>0</td>\n",
       "      <td>IL</td>\n",
       "      <td>61081</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>3712</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>L</td>\n",
       "      <td>4</td>\n",
       "      <td>E</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>39.0</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9401</td>\n",
       "      <td>BOA</td>\n",
       "      <td>1</td>\n",
       "      <td>CA</td>\n",
       "      <td>91326</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>5202</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>L</td>\n",
       "      <td>2</td>\n",
       "      <td>G</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>1.0</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9001</td>\n",
       "      <td>AMH</td>\n",
       "      <td>1</td>\n",
       "      <td>NC</td>\n",
       "      <td>27017</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>L</td>\n",
       "      <td>4</td>\n",
       "      <td>E</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>60.0</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8701</td>\n",
       "      <td>BRY</td>\n",
       "      <td>0</td>\n",
       "      <td>CA</td>\n",
       "      <td>95953</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>2801</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>L</td>\n",
       "      <td>4</td>\n",
       "      <td>E</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>41.0</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>8601</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>FL</td>\n",
       "      <td>33176</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>2001</td>\n",
       "      <td>0</td>\n",
       "      <td>X</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>L</td>\n",
       "      <td>2</td>\n",
       "      <td>F</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>26.0</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 481 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   ODATEDW OSOURCE  TCODE STATE    ZIP MAILCODE PVASTATE   DOB NOEXCH  \\\n",
       "0     8901     GRI      0    IL  61081                    3712      0   \n",
       "1     9401     BOA      1    CA  91326                    5202      0   \n",
       "2     9001     AMH      1    NC  27017                       0      0   \n",
       "3     8701     BRY      0    CA  95953                    2801      0   \n",
       "4     8601              0    FL  33176                    2001      0   \n",
       "\n",
       "  RECINHSE  ... TARGET_D HPHONE_D RFA_2R RFA_2F RFA_2A MDMAUD_R  MDMAUD_F  \\\n",
       "0           ...      0.0        0      L      4      E        X         X   \n",
       "1           ...      0.0        0      L      2      G        X         X   \n",
       "2           ...      0.0        1      L      4      E        X         X   \n",
       "3           ...      0.0        1      L      4      E        X         X   \n",
       "4        X  ...      0.0        1      L      2      F        X         X   \n",
       "\n",
       "  MDMAUD_A CLUSTER2 GEOCODE2  \n",
       "0        X     39.0        C  \n",
       "1        X      1.0        A  \n",
       "2        X     60.0        C  \n",
       "3        X     41.0        C  \n",
       "4        X     26.0        A  \n",
       "\n",
       "[5 rows x 481 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "0406957f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(95412, 481)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "79185bb5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ODATEDW</th>\n",
       "      <th>TCODE</th>\n",
       "      <th>DOB</th>\n",
       "      <th>AGE</th>\n",
       "      <th>NUMCHLD</th>\n",
       "      <th>INCOME</th>\n",
       "      <th>WEALTH1</th>\n",
       "      <th>HIT</th>\n",
       "      <th>MBCRAFT</th>\n",
       "      <th>MBGARDEN</th>\n",
       "      <th>...</th>\n",
       "      <th>FISTDATE</th>\n",
       "      <th>NEXTDATE</th>\n",
       "      <th>TIMELAG</th>\n",
       "      <th>AVGGIFT</th>\n",
       "      <th>CONTROLN</th>\n",
       "      <th>TARGET_B</th>\n",
       "      <th>TARGET_D</th>\n",
       "      <th>HPHONE_D</th>\n",
       "      <th>RFA_2F</th>\n",
       "      <th>CLUSTER2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>95412.000000</td>\n",
       "      <td>95412.000000</td>\n",
       "      <td>95412.000000</td>\n",
       "      <td>71747.000000</td>\n",
       "      <td>12386.000000</td>\n",
       "      <td>74126.000000</td>\n",
       "      <td>50680.000000</td>\n",
       "      <td>95412.000000</td>\n",
       "      <td>42558.000000</td>\n",
       "      <td>42558.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>95412.000000</td>\n",
       "      <td>85439.000000</td>\n",
       "      <td>85439.000000</td>\n",
       "      <td>95412.000000</td>\n",
       "      <td>95412.000000</td>\n",
       "      <td>95412.000000</td>\n",
       "      <td>95412.000000</td>\n",
       "      <td>95412.000000</td>\n",
       "      <td>95412.000000</td>\n",
       "      <td>95280.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>9141.363256</td>\n",
       "      <td>54.223117</td>\n",
       "      <td>2723.602933</td>\n",
       "      <td>61.611649</td>\n",
       "      <td>1.527773</td>\n",
       "      <td>3.886248</td>\n",
       "      <td>5.345699</td>\n",
       "      <td>3.321438</td>\n",
       "      <td>0.152075</td>\n",
       "      <td>0.059166</td>\n",
       "      <td>...</td>\n",
       "      <td>9135.651648</td>\n",
       "      <td>9151.022917</td>\n",
       "      <td>8.093739</td>\n",
       "      <td>13.347786</td>\n",
       "      <td>95778.176959</td>\n",
       "      <td>0.050759</td>\n",
       "      <td>0.793073</td>\n",
       "      <td>0.500618</td>\n",
       "      <td>1.910053</td>\n",
       "      <td>31.533711</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>343.454752</td>\n",
       "      <td>953.844476</td>\n",
       "      <td>2132.241295</td>\n",
       "      <td>16.664199</td>\n",
       "      <td>0.806861</td>\n",
       "      <td>1.854960</td>\n",
       "      <td>2.742490</td>\n",
       "      <td>9.306899</td>\n",
       "      <td>0.470023</td>\n",
       "      <td>0.262078</td>\n",
       "      <td>...</td>\n",
       "      <td>320.394019</td>\n",
       "      <td>294.257260</td>\n",
       "      <td>8.213242</td>\n",
       "      <td>10.769997</td>\n",
       "      <td>55284.596094</td>\n",
       "      <td>0.219506</td>\n",
       "      <td>4.429725</td>\n",
       "      <td>0.500002</td>\n",
       "      <td>1.072749</td>\n",
       "      <td>18.764614</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>8306.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7211.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.285714</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>8801.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>201.000000</td>\n",
       "      <td>48.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>8810.000000</td>\n",
       "      <td>8903.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>8.384615</td>\n",
       "      <td>47910.750000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>15.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>9201.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2610.000000</td>\n",
       "      <td>62.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>9201.000000</td>\n",
       "      <td>9204.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>11.636364</td>\n",
       "      <td>95681.500000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>32.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>9501.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>4601.000000</td>\n",
       "      <td>75.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>9409.000000</td>\n",
       "      <td>9409.000000</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>15.477955</td>\n",
       "      <td>143643.500000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>49.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>9701.000000</td>\n",
       "      <td>72002.000000</td>\n",
       "      <td>9710.000000</td>\n",
       "      <td>98.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>241.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>9603.000000</td>\n",
       "      <td>9702.000000</td>\n",
       "      <td>1088.000000</td>\n",
       "      <td>1000.000000</td>\n",
       "      <td>191779.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>200.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>62.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 407 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            ODATEDW         TCODE           DOB           AGE       NUMCHLD  \\\n",
       "count  95412.000000  95412.000000  95412.000000  71747.000000  12386.000000   \n",
       "mean    9141.363256     54.223117   2723.602933     61.611649      1.527773   \n",
       "std      343.454752    953.844476   2132.241295     16.664199      0.806861   \n",
       "min     8306.000000      0.000000      0.000000      1.000000      1.000000   \n",
       "25%     8801.000000      0.000000    201.000000     48.000000      1.000000   \n",
       "50%     9201.000000      1.000000   2610.000000     62.000000      1.000000   \n",
       "75%     9501.000000      2.000000   4601.000000     75.000000      2.000000   \n",
       "max     9701.000000  72002.000000   9710.000000     98.000000      7.000000   \n",
       "\n",
       "             INCOME       WEALTH1           HIT       MBCRAFT      MBGARDEN  \\\n",
       "count  74126.000000  50680.000000  95412.000000  42558.000000  42558.000000   \n",
       "mean       3.886248      5.345699      3.321438      0.152075      0.059166   \n",
       "std        1.854960      2.742490      9.306899      0.470023      0.262078   \n",
       "min        1.000000      0.000000      0.000000      0.000000      0.000000   \n",
       "25%        2.000000      3.000000      0.000000      0.000000      0.000000   \n",
       "50%        4.000000      6.000000      0.000000      0.000000      0.000000   \n",
       "75%        5.000000      8.000000      3.000000      0.000000      0.000000   \n",
       "max        7.000000      9.000000    241.000000      6.000000      4.000000   \n",
       "\n",
       "       ...      FISTDATE      NEXTDATE       TIMELAG       AVGGIFT  \\\n",
       "count  ...  95412.000000  85439.000000  85439.000000  95412.000000   \n",
       "mean   ...   9135.651648   9151.022917      8.093739     13.347786   \n",
       "std    ...    320.394019    294.257260      8.213242     10.769997   \n",
       "min    ...      0.000000   7211.000000      0.000000      1.285714   \n",
       "25%    ...   8810.000000   8903.000000      4.000000      8.384615   \n",
       "50%    ...   9201.000000   9204.000000      6.000000     11.636364   \n",
       "75%    ...   9409.000000   9409.000000     11.000000     15.477955   \n",
       "max    ...   9603.000000   9702.000000   1088.000000   1000.000000   \n",
       "\n",
       "            CONTROLN      TARGET_B      TARGET_D      HPHONE_D        RFA_2F  \\\n",
       "count   95412.000000  95412.000000  95412.000000  95412.000000  95412.000000   \n",
       "mean    95778.176959      0.050759      0.793073      0.500618      1.910053   \n",
       "std     55284.596094      0.219506      4.429725      0.500002      1.072749   \n",
       "min         1.000000      0.000000      0.000000      0.000000      1.000000   \n",
       "25%     47910.750000      0.000000      0.000000      0.000000      1.000000   \n",
       "50%     95681.500000      0.000000      0.000000      1.000000      2.000000   \n",
       "75%    143643.500000      0.000000      0.000000      1.000000      3.000000   \n",
       "max    191779.000000      1.000000    200.000000      1.000000      4.000000   \n",
       "\n",
       "           CLUSTER2  \n",
       "count  95280.000000  \n",
       "mean      31.533711  \n",
       "std       18.764614  \n",
       "min        1.000000  \n",
       "25%       15.000000  \n",
       "50%       32.000000  \n",
       "75%       49.000000  \n",
       "max       62.000000  \n",
       "\n",
       "[8 rows x 407 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "094b07fa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RDATE_5     95403\n",
       "RAMNT_5     95403\n",
       "RDATE_3     95170\n",
       "RAMNT_3     95170\n",
       "RDATE_4     95131\n",
       "RAMNT_4     95131\n",
       "RAMNT_6     94636\n",
       "RDATE_6     94636\n",
       "RAMNT_15    88150\n",
       "RDATE_15    88150\n",
       "RDATE_23    87553\n",
       "RAMNT_23    87553\n",
       "RDATE_20    87524\n",
       "RAMNT_20    87524\n",
       "RAMNT_7     86517\n",
       "RDATE_7     86517\n",
       "RAMNT_17    86011\n",
       "RDATE_17    86011\n",
       "RDATE_21    85899\n",
       "RAMNT_21    85899\n",
       "RAMNT_10    84951\n",
       "RDATE_10    84951\n",
       "RDATE_13    83162\n",
       "RAMNT_13    83162\n",
       "NUMCHLD     83026\n",
       "RDATE_11    80672\n",
       "RAMNT_11    80672\n",
       "RDATE_19    79535\n",
       "RAMNT_19    79535\n",
       "RAMNT_9     78678\n",
       "RDATE_9     78678\n",
       "RDATE_24    77674\n",
       "RAMNT_24    77674\n",
       "RAMNT_18    75634\n",
       "RDATE_18    75634\n",
       "RAMNT_22    74539\n",
       "RDATE_22    74539\n",
       "RAMNT_8     73940\n",
       "RDATE_8     73940\n",
       "RDATE_14    72095\n",
       "RAMNT_14    72095\n",
       "RAMNT_12    69712\n",
       "RDATE_12    69712\n",
       "RAMNT_16    68418\n",
       "RDATE_16    68418\n",
       "ADATE_15    65477\n",
       "ADATE_23    56270\n",
       "MBCOLECT    52914\n",
       "MAGFAML     52854\n",
       "MBCRAFT     52854\n",
       "dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file.isna().sum().sort_values(ascending = False).head(50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "71f12dce",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_numerical = file.select_dtypes(include='int64')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "85878934",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_cat = file.select_dtypes(include='object')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "dbeda5b3",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ODATEDW</th>\n",
       "      <th>TCODE</th>\n",
       "      <th>DOB</th>\n",
       "      <th>HIT</th>\n",
       "      <th>MALEMILI</th>\n",
       "      <th>MALEVET</th>\n",
       "      <th>VIETVETS</th>\n",
       "      <th>WWIIVETS</th>\n",
       "      <th>LOCALGOV</th>\n",
       "      <th>STATEGOV</th>\n",
       "      <th>...</th>\n",
       "      <th>NGIFTALL</th>\n",
       "      <th>CARDGIFT</th>\n",
       "      <th>MINRDATE</th>\n",
       "      <th>MAXRDATE</th>\n",
       "      <th>LASTDATE</th>\n",
       "      <th>FISTDATE</th>\n",
       "      <th>CONTROLN</th>\n",
       "      <th>TARGET_B</th>\n",
       "      <th>HPHONE_D</th>\n",
       "      <th>RFA_2F</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8901</td>\n",
       "      <td>0</td>\n",
       "      <td>3712</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>39</td>\n",
       "      <td>34</td>\n",
       "      <td>18</td>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>31</td>\n",
       "      <td>14</td>\n",
       "      <td>9208</td>\n",
       "      <td>9402</td>\n",
       "      <td>9512</td>\n",
       "      <td>8911</td>\n",
       "      <td>95515</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9401</td>\n",
       "      <td>1</td>\n",
       "      <td>5202</td>\n",
       "      <td>16</td>\n",
       "      <td>0</td>\n",
       "      <td>15</td>\n",
       "      <td>55</td>\n",
       "      <td>11</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>9310</td>\n",
       "      <td>9512</td>\n",
       "      <td>9512</td>\n",
       "      <td>9310</td>\n",
       "      <td>148535</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9001</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>20</td>\n",
       "      <td>29</td>\n",
       "      <td>33</td>\n",
       "      <td>6</td>\n",
       "      <td>8</td>\n",
       "      <td>...</td>\n",
       "      <td>27</td>\n",
       "      <td>14</td>\n",
       "      <td>9111</td>\n",
       "      <td>9207</td>\n",
       "      <td>9512</td>\n",
       "      <td>9001</td>\n",
       "      <td>15078</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8701</td>\n",
       "      <td>0</td>\n",
       "      <td>2801</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>23</td>\n",
       "      <td>14</td>\n",
       "      <td>31</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>16</td>\n",
       "      <td>7</td>\n",
       "      <td>8711</td>\n",
       "      <td>9411</td>\n",
       "      <td>9512</td>\n",
       "      <td>8702</td>\n",
       "      <td>172556</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>8601</td>\n",
       "      <td>0</td>\n",
       "      <td>2001</td>\n",
       "      <td>60</td>\n",
       "      <td>1</td>\n",
       "      <td>28</td>\n",
       "      <td>9</td>\n",
       "      <td>53</td>\n",
       "      <td>26</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>37</td>\n",
       "      <td>8</td>\n",
       "      <td>9310</td>\n",
       "      <td>9601</td>\n",
       "      <td>9601</td>\n",
       "      <td>7903</td>\n",
       "      <td>7112</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95407</th>\n",
       "      <td>9601</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>14</td>\n",
       "      <td>36</td>\n",
       "      <td>47</td>\n",
       "      <td>11</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>9602</td>\n",
       "      <td>9602</td>\n",
       "      <td>9602</td>\n",
       "      <td>9602</td>\n",
       "      <td>184568</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95408</th>\n",
       "      <td>9601</td>\n",
       "      <td>1</td>\n",
       "      <td>5001</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>31</td>\n",
       "      <td>43</td>\n",
       "      <td>19</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>9603</td>\n",
       "      <td>9603</td>\n",
       "      <td>9603</td>\n",
       "      <td>9603</td>\n",
       "      <td>122706</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95409</th>\n",
       "      <td>9501</td>\n",
       "      <td>1</td>\n",
       "      <td>3801</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>18</td>\n",
       "      <td>46</td>\n",
       "      <td>20</td>\n",
       "      <td>7</td>\n",
       "      <td>23</td>\n",
       "      <td>...</td>\n",
       "      <td>7</td>\n",
       "      <td>4</td>\n",
       "      <td>9603</td>\n",
       "      <td>9501</td>\n",
       "      <td>9610</td>\n",
       "      <td>9410</td>\n",
       "      <td>189641</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95410</th>\n",
       "      <td>8601</td>\n",
       "      <td>0</td>\n",
       "      <td>4005</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>28</td>\n",
       "      <td>35</td>\n",
       "      <td>20</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>41</td>\n",
       "      <td>18</td>\n",
       "      <td>9011</td>\n",
       "      <td>9608</td>\n",
       "      <td>9701</td>\n",
       "      <td>8612</td>\n",
       "      <td>4693</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95411</th>\n",
       "      <td>8801</td>\n",
       "      <td>2</td>\n",
       "      <td>1801</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>33</td>\n",
       "      <td>22</td>\n",
       "      <td>27</td>\n",
       "      <td>8</td>\n",
       "      <td>10</td>\n",
       "      <td>...</td>\n",
       "      <td>39</td>\n",
       "      <td>11</td>\n",
       "      <td>8812</td>\n",
       "      <td>9312</td>\n",
       "      <td>9612</td>\n",
       "      <td>8803</td>\n",
       "      <td>185114</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>95412 rows × 310 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       ODATEDW  TCODE   DOB  HIT  MALEMILI  MALEVET  VIETVETS  WWIIVETS  \\\n",
       "0         8901      0  3712    0         0       39        34        18   \n",
       "1         9401      1  5202   16         0       15        55        11   \n",
       "2         9001      1     0    2         0       20        29        33   \n",
       "3         8701      0  2801    2         0       23        14        31   \n",
       "4         8601      0  2001   60         1       28         9        53   \n",
       "...        ...    ...   ...  ...       ...      ...       ...       ...   \n",
       "95407     9601      1     0    0        14       36        47        11   \n",
       "95408     9601      1  5001    1         0       31        43        19   \n",
       "95409     9501      1  3801    0         0       18        46        20   \n",
       "95410     8601      0  4005    0         0       28        35        20   \n",
       "95411     8801      2  1801    3         1       33        22        27   \n",
       "\n",
       "       LOCALGOV  STATEGOV  ...  NGIFTALL  CARDGIFT  MINRDATE  MAXRDATE  \\\n",
       "0            10         2  ...        31        14      9208      9402   \n",
       "1             6         2  ...         3         1      9310      9512   \n",
       "2             6         8  ...        27        14      9111      9207   \n",
       "3             3         0  ...        16         7      8711      9411   \n",
       "4            26         3  ...        37         8      9310      9601   \n",
       "...         ...       ...  ...       ...       ...       ...       ...   \n",
       "95407         7         8  ...         1         0      9602      9602   \n",
       "95408         4         1  ...         1         0      9603      9603   \n",
       "95409         7        23  ...         7         4      9603      9501   \n",
       "95410         9         1  ...        41        18      9011      9608   \n",
       "95411         8        10  ...        39        11      8812      9312   \n",
       "\n",
       "       LASTDATE  FISTDATE  CONTROLN  TARGET_B  HPHONE_D  RFA_2F  \n",
       "0          9512      8911     95515         0         0       4  \n",
       "1          9512      9310    148535         0         0       2  \n",
       "2          9512      9001     15078         0         1       4  \n",
       "3          9512      8702    172556         0         1       4  \n",
       "4          9601      7903      7112         0         1       2  \n",
       "...         ...       ...       ...       ...       ...     ...  \n",
       "95407      9602      9602    184568         0         0       1  \n",
       "95408      9603      9603    122706         0         1       1  \n",
       "95409      9610      9410    189641         0         1       3  \n",
       "95410      9701      8612      4693         1         1       4  \n",
       "95411      9612      8803    185114         0         1       1  \n",
       "\n",
       "[95412 rows x 310 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_numerical "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "0665760f",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>OSOURCE</th>\n",
       "      <th>STATE</th>\n",
       "      <th>ZIP</th>\n",
       "      <th>MAILCODE</th>\n",
       "      <th>PVASTATE</th>\n",
       "      <th>NOEXCH</th>\n",
       "      <th>RECINHSE</th>\n",
       "      <th>RECP3</th>\n",
       "      <th>RECPGVG</th>\n",
       "      <th>RECSWEEP</th>\n",
       "      <th>...</th>\n",
       "      <th>RFA_21</th>\n",
       "      <th>RFA_22</th>\n",
       "      <th>RFA_23</th>\n",
       "      <th>RFA_24</th>\n",
       "      <th>RFA_2R</th>\n",
       "      <th>RFA_2A</th>\n",
       "      <th>MDMAUD_R</th>\n",
       "      <th>MDMAUD_F</th>\n",
       "      <th>MDMAUD_A</th>\n",
       "      <th>GEOCODE2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>GRI</td>\n",
       "      <td>IL</td>\n",
       "      <td>61081</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>S4E</td>\n",
       "      <td>S4E</td>\n",
       "      <td>S4E</td>\n",
       "      <td>S4E</td>\n",
       "      <td>L</td>\n",
       "      <td>E</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>BOA</td>\n",
       "      <td>CA</td>\n",
       "      <td>91326</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>N1E</td>\n",
       "      <td>N1E</td>\n",
       "      <td></td>\n",
       "      <td>F1E</td>\n",
       "      <td>L</td>\n",
       "      <td>G</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AMH</td>\n",
       "      <td>NC</td>\n",
       "      <td>27017</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td></td>\n",
       "      <td>S4D</td>\n",
       "      <td>S4D</td>\n",
       "      <td>S3D</td>\n",
       "      <td>L</td>\n",
       "      <td>E</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>BRY</td>\n",
       "      <td>CA</td>\n",
       "      <td>95953</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>A1D</td>\n",
       "      <td>A1D</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>L</td>\n",
       "      <td>E</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td></td>\n",
       "      <td>FL</td>\n",
       "      <td>33176</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>A3D</td>\n",
       "      <td>I4E</td>\n",
       "      <td>A3D</td>\n",
       "      <td>A3D</td>\n",
       "      <td>L</td>\n",
       "      <td>F</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95407</th>\n",
       "      <td>ASE</td>\n",
       "      <td>AK</td>\n",
       "      <td>99504</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>L</td>\n",
       "      <td>G</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95408</th>\n",
       "      <td>DCD</td>\n",
       "      <td>TX</td>\n",
       "      <td>77379</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>L</td>\n",
       "      <td>F</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95409</th>\n",
       "      <td>MBC</td>\n",
       "      <td>MI</td>\n",
       "      <td>48910</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td>X</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>P1D</td>\n",
       "      <td>P1D</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>L</td>\n",
       "      <td>E</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95410</th>\n",
       "      <td>PRV</td>\n",
       "      <td>CA</td>\n",
       "      <td>91320</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>X</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>A1F</td>\n",
       "      <td>A1F</td>\n",
       "      <td>S2F</td>\n",
       "      <td>S3F</td>\n",
       "      <td>L</td>\n",
       "      <td>F</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95411</th>\n",
       "      <td>MCC</td>\n",
       "      <td>NC</td>\n",
       "      <td>28409</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>X</td>\n",
       "      <td></td>\n",
       "      <td>X</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>S3G</td>\n",
       "      <td>S3G</td>\n",
       "      <td>S3G</td>\n",
       "      <td>S3G</td>\n",
       "      <td>L</td>\n",
       "      <td>G</td>\n",
       "      <td>C</td>\n",
       "      <td>1</td>\n",
       "      <td>C</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>95412 rows × 74 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      OSOURCE STATE    ZIP MAILCODE PVASTATE NOEXCH RECINHSE RECP3 RECPGVG  \\\n",
       "0         GRI    IL  61081                        0                          \n",
       "1         BOA    CA  91326                        0                          \n",
       "2         AMH    NC  27017                        0                          \n",
       "3         BRY    CA  95953                        0                          \n",
       "4                FL  33176                        0        X     X           \n",
       "...       ...   ...    ...      ...      ...    ...      ...   ...     ...   \n",
       "95407     ASE    AK  99504                        0                          \n",
       "95408     DCD    TX  77379                        0                          \n",
       "95409     MBC    MI  48910                        0              X           \n",
       "95410     PRV    CA  91320                        0        X                 \n",
       "95411     MCC    NC  28409                        0        X             X   \n",
       "\n",
       "      RECSWEEP  ... RFA_21 RFA_22 RFA_23 RFA_24 RFA_2R RFA_2A MDMAUD_R  \\\n",
       "0               ...    S4E    S4E    S4E    S4E      L      E        X   \n",
       "1               ...    N1E    N1E           F1E      L      G        X   \n",
       "2               ...           S4D    S4D    S3D      L      E        X   \n",
       "3               ...    A1D    A1D                    L      E        X   \n",
       "4               ...    A3D    I4E    A3D    A3D      L      F        X   \n",
       "...        ...  ...    ...    ...    ...    ...    ...    ...      ...   \n",
       "95407           ...                                  L      G        X   \n",
       "95408           ...                                  L      F        X   \n",
       "95409           ...    P1D    P1D                    L      E        X   \n",
       "95410           ...    A1F    A1F    S2F    S3F      L      F        X   \n",
       "95411           ...    S3G    S3G    S3G    S3G      L      G        C   \n",
       "\n",
       "      MDMAUD_F MDMAUD_A GEOCODE2  \n",
       "0            X        X        C  \n",
       "1            X        X        A  \n",
       "2            X        X        C  \n",
       "3            X        X        C  \n",
       "4            X        X        A  \n",
       "...        ...      ...      ...  \n",
       "95407        X        X        C  \n",
       "95408        X        X        A  \n",
       "95409        X        X        B  \n",
       "95410        X        X        A  \n",
       "95411        1        C        C  \n",
       "\n",
       "[95412 rows x 74 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_cat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "904b4252",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TARGET_B    1.000000\n",
       "TARGET_D    0.774232\n",
       "RFA_2F      0.072311\n",
       "CARDGIFT    0.054027\n",
       "NGIFTALL    0.050896\n",
       "              ...   \n",
       "RAMNT_3    -0.095351\n",
       "RDATE_3    -0.126060\n",
       "RAMNT_5    -0.380296\n",
       "ADATE_5          NaN\n",
       "ADATE_15         NaN\n",
       "Name: TARGET_B, Length: 407, dtype: float64"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file_corr = file.corr()['TARGET_B'].sort_values(ascending=False)\n",
    "file_corr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "9416329f",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>percentage</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>RDATE_5</th>\n",
       "      <td>99.990567</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RAMNT_5</th>\n",
       "      <td>99.990567</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RDATE_3</th>\n",
       "      <td>99.746363</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RAMNT_3</th>\n",
       "      <td>99.746363</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RDATE_4</th>\n",
       "      <td>99.705488</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ADATE_16</th>\n",
       "      <td>21.343227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ADATE_14</th>\n",
       "      <td>19.774242</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ADATE_9</th>\n",
       "      <td>11.785729</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ADATE_11</th>\n",
       "      <td>10.923154</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NEXTDATE</th>\n",
       "      <td>10.452564</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>80 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          percentage\n",
       "RDATE_5    99.990567\n",
       "RAMNT_5    99.990567\n",
       "RDATE_3    99.746363\n",
       "RAMNT_3    99.746363\n",
       "RDATE_4    99.705488\n",
       "...              ...\n",
       "ADATE_16   21.343227\n",
       "ADATE_14   19.774242\n",
       "ADATE_9    11.785729\n",
       "ADATE_11   10.923154\n",
       "NEXTDATE   10.452564\n",
       "\n",
       "[80 rows x 1 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nulls = pd.DataFrame(file.isna().sum()*100/len(file), columns=['percentage'])\n",
    "nulls.sort_values('percentage', ascending = False).head(80)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5233d217",
   "metadata": {},
   "source": [
    "Data Cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "73421971",
   "metadata": {},
   "outputs": [],
   "source": [
    "#After checking the internet\n",
    "#I have decided to check the internet, what to do with dataframes with a lot of columns. \n",
    "#I saw that I could drop columns where the data in the columns are more than 10%"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "710abadc",
   "metadata": {},
   "outputs": [],
   "source": [
    "less_ten_10_percent = list(nulls[nulls['percentage'] < 10].index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "b9ac1c09",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['ODATEDW',\n",
       " 'OSOURCE',\n",
       " 'TCODE',\n",
       " 'STATE',\n",
       " 'ZIP',\n",
       " 'MAILCODE',\n",
       " 'PVASTATE',\n",
       " 'DOB',\n",
       " 'NOEXCH',\n",
       " 'RECINHSE',\n",
       " 'RECP3',\n",
       " 'RECPGVG',\n",
       " 'RECSWEEP',\n",
       " 'MDMAUD',\n",
       " 'DOMAIN',\n",
       " 'CLUSTER',\n",
       " 'AGEFLAG',\n",
       " 'HOMEOWNR',\n",
       " 'CHILD03',\n",
       " 'CHILD07',\n",
       " 'CHILD12',\n",
       " 'CHILD18',\n",
       " 'GENDER',\n",
       " 'HIT',\n",
       " 'DATASRCE',\n",
       " 'MALEMILI',\n",
       " 'MALEVET',\n",
       " 'VIETVETS',\n",
       " 'WWIIVETS',\n",
       " 'LOCALGOV',\n",
       " 'STATEGOV',\n",
       " 'FEDGOV',\n",
       " 'SOLP3',\n",
       " 'SOLIH',\n",
       " 'MAJOR',\n",
       " 'GEOCODE',\n",
       " 'COLLECT1',\n",
       " 'VETERANS',\n",
       " 'BIBLE',\n",
       " 'CATLG',\n",
       " 'HOMEE',\n",
       " 'PETS',\n",
       " 'CDPLAY',\n",
       " 'STEREO',\n",
       " 'PCOWNERS',\n",
       " 'PHOTO',\n",
       " 'CRAFTS',\n",
       " 'FISHER',\n",
       " 'GARDENIN',\n",
       " 'BOATS',\n",
       " 'WALKER',\n",
       " 'KIDSTUFF',\n",
       " 'CARDS',\n",
       " 'PLATES',\n",
       " 'LIFESRC',\n",
       " 'PEPSTRFL',\n",
       " 'POP901',\n",
       " 'POP902',\n",
       " 'POP903',\n",
       " 'POP90C1',\n",
       " 'POP90C2',\n",
       " 'POP90C3',\n",
       " 'POP90C4',\n",
       " 'POP90C5',\n",
       " 'ETH1',\n",
       " 'ETH2',\n",
       " 'ETH3',\n",
       " 'ETH4',\n",
       " 'ETH5',\n",
       " 'ETH6',\n",
       " 'ETH7',\n",
       " 'ETH8',\n",
       " 'ETH9',\n",
       " 'ETH10',\n",
       " 'ETH11',\n",
       " 'ETH12',\n",
       " 'ETH13',\n",
       " 'ETH14',\n",
       " 'ETH15',\n",
       " 'ETH16',\n",
       " 'AGE901',\n",
       " 'AGE902',\n",
       " 'AGE903',\n",
       " 'AGE904',\n",
       " 'AGE905',\n",
       " 'AGE906',\n",
       " 'AGE907',\n",
       " 'CHIL1',\n",
       " 'CHIL2',\n",
       " 'CHIL3',\n",
       " 'AGEC1',\n",
       " 'AGEC2',\n",
       " 'AGEC3',\n",
       " 'AGEC4',\n",
       " 'AGEC5',\n",
       " 'AGEC6',\n",
       " 'AGEC7',\n",
       " 'CHILC1',\n",
       " 'CHILC2',\n",
       " 'CHILC3',\n",
       " 'CHILC4',\n",
       " 'CHILC5',\n",
       " 'HHAGE1',\n",
       " 'HHAGE2',\n",
       " 'HHAGE3',\n",
       " 'HHN1',\n",
       " 'HHN2',\n",
       " 'HHN3',\n",
       " 'HHN4',\n",
       " 'HHN5',\n",
       " 'HHN6',\n",
       " 'MARR1',\n",
       " 'MARR2',\n",
       " 'MARR3',\n",
       " 'MARR4',\n",
       " 'HHP1',\n",
       " 'HHP2',\n",
       " 'DW1',\n",
       " 'DW2',\n",
       " 'DW3',\n",
       " 'DW4',\n",
       " 'DW5',\n",
       " 'DW6',\n",
       " 'DW7',\n",
       " 'DW8',\n",
       " 'DW9',\n",
       " 'HV1',\n",
       " 'HV2',\n",
       " 'HV3',\n",
       " 'HV4',\n",
       " 'HU1',\n",
       " 'HU2',\n",
       " 'HU3',\n",
       " 'HU4',\n",
       " 'HU5',\n",
       " 'HHD1',\n",
       " 'HHD2',\n",
       " 'HHD3',\n",
       " 'HHD4',\n",
       " 'HHD5',\n",
       " 'HHD6',\n",
       " 'HHD7',\n",
       " 'HHD8',\n",
       " 'HHD9',\n",
       " 'HHD10',\n",
       " 'HHD11',\n",
       " 'HHD12',\n",
       " 'ETHC1',\n",
       " 'ETHC2',\n",
       " 'ETHC3',\n",
       " 'ETHC4',\n",
       " 'ETHC5',\n",
       " 'ETHC6',\n",
       " 'HVP1',\n",
       " 'HVP2',\n",
       " 'HVP3',\n",
       " 'HVP4',\n",
       " 'HVP5',\n",
       " 'HVP6',\n",
       " 'HUR1',\n",
       " 'HUR2',\n",
       " 'RHP1',\n",
       " 'RHP2',\n",
       " 'RHP3',\n",
       " 'RHP4',\n",
       " 'HUPA1',\n",
       " 'HUPA2',\n",
       " 'HUPA3',\n",
       " 'HUPA4',\n",
       " 'HUPA5',\n",
       " 'HUPA6',\n",
       " 'HUPA7',\n",
       " 'RP1',\n",
       " 'RP2',\n",
       " 'RP3',\n",
       " 'RP4',\n",
       " 'MSA',\n",
       " 'ADI',\n",
       " 'DMA',\n",
       " 'IC1',\n",
       " 'IC2',\n",
       " 'IC3',\n",
       " 'IC4',\n",
       " 'IC5',\n",
       " 'IC6',\n",
       " 'IC7',\n",
       " 'IC8',\n",
       " 'IC9',\n",
       " 'IC10',\n",
       " 'IC11',\n",
       " 'IC12',\n",
       " 'IC13',\n",
       " 'IC14',\n",
       " 'IC15',\n",
       " 'IC16',\n",
       " 'IC17',\n",
       " 'IC18',\n",
       " 'IC19',\n",
       " 'IC20',\n",
       " 'IC21',\n",
       " 'IC22',\n",
       " 'IC23',\n",
       " 'HHAS1',\n",
       " 'HHAS2',\n",
       " 'HHAS3',\n",
       " 'HHAS4',\n",
       " 'MC1',\n",
       " 'MC2',\n",
       " 'MC3',\n",
       " 'TPE1',\n",
       " 'TPE2',\n",
       " 'TPE3',\n",
       " 'TPE4',\n",
       " 'TPE5',\n",
       " 'TPE6',\n",
       " 'TPE7',\n",
       " 'TPE8',\n",
       " 'TPE9',\n",
       " 'PEC1',\n",
       " 'PEC2',\n",
       " 'TPE10',\n",
       " 'TPE11',\n",
       " 'TPE12',\n",
       " 'TPE13',\n",
       " 'LFC1',\n",
       " 'LFC2',\n",
       " 'LFC3',\n",
       " 'LFC4',\n",
       " 'LFC5',\n",
       " 'LFC6',\n",
       " 'LFC7',\n",
       " 'LFC8',\n",
       " 'LFC9',\n",
       " 'LFC10',\n",
       " 'OCC1',\n",
       " 'OCC2',\n",
       " 'OCC3',\n",
       " 'OCC4',\n",
       " 'OCC5',\n",
       " 'OCC6',\n",
       " 'OCC7',\n",
       " 'OCC8',\n",
       " 'OCC9',\n",
       " 'OCC10',\n",
       " 'OCC11',\n",
       " 'OCC12',\n",
       " 'OCC13',\n",
       " 'EIC1',\n",
       " 'EIC2',\n",
       " 'EIC3',\n",
       " 'EIC4',\n",
       " 'EIC5',\n",
       " 'EIC6',\n",
       " 'EIC7',\n",
       " 'EIC8',\n",
       " 'EIC9',\n",
       " 'EIC10',\n",
       " 'EIC11',\n",
       " 'EIC12',\n",
       " 'EIC13',\n",
       " 'EIC14',\n",
       " 'EIC15',\n",
       " 'EIC16',\n",
       " 'OEDC1',\n",
       " 'OEDC2',\n",
       " 'OEDC3',\n",
       " 'OEDC4',\n",
       " 'OEDC5',\n",
       " 'OEDC6',\n",
       " 'OEDC7',\n",
       " 'EC1',\n",
       " 'EC2',\n",
       " 'EC3',\n",
       " 'EC4',\n",
       " 'EC5',\n",
       " 'EC6',\n",
       " 'EC7',\n",
       " 'EC8',\n",
       " 'SEC1',\n",
       " 'SEC2',\n",
       " 'SEC3',\n",
       " 'SEC4',\n",
       " 'SEC5',\n",
       " 'AFC1',\n",
       " 'AFC2',\n",
       " 'AFC3',\n",
       " 'AFC4',\n",
       " 'AFC5',\n",
       " 'AFC6',\n",
       " 'VC1',\n",
       " 'VC2',\n",
       " 'VC3',\n",
       " 'VC4',\n",
       " 'ANC1',\n",
       " 'ANC2',\n",
       " 'ANC3',\n",
       " 'ANC4',\n",
       " 'ANC5',\n",
       " 'ANC6',\n",
       " 'ANC7',\n",
       " 'ANC8',\n",
       " 'ANC9',\n",
       " 'ANC10',\n",
       " 'ANC11',\n",
       " 'ANC12',\n",
       " 'ANC13',\n",
       " 'ANC14',\n",
       " 'ANC15',\n",
       " 'POBC1',\n",
       " 'POBC2',\n",
       " 'LSC1',\n",
       " 'LSC2',\n",
       " 'LSC3',\n",
       " 'LSC4',\n",
       " 'VOC1',\n",
       " 'VOC2',\n",
       " 'VOC3',\n",
       " 'HC1',\n",
       " 'HC2',\n",
       " 'HC3',\n",
       " 'HC4',\n",
       " 'HC5',\n",
       " 'HC6',\n",
       " 'HC7',\n",
       " 'HC8',\n",
       " 'HC9',\n",
       " 'HC10',\n",
       " 'HC11',\n",
       " 'HC12',\n",
       " 'HC13',\n",
       " 'HC14',\n",
       " 'HC15',\n",
       " 'HC16',\n",
       " 'HC17',\n",
       " 'HC18',\n",
       " 'HC19',\n",
       " 'HC20',\n",
       " 'HC21',\n",
       " 'MHUC1',\n",
       " 'MHUC2',\n",
       " 'AC1',\n",
       " 'AC2',\n",
       " 'ADATE_2',\n",
       " 'ADATE_3',\n",
       " 'ADATE_4',\n",
       " 'ADATE_6',\n",
       " 'ADATE_7',\n",
       " 'ADATE_8',\n",
       " 'ADATE_12',\n",
       " 'RFA_2',\n",
       " 'RFA_3',\n",
       " 'RFA_4',\n",
       " 'RFA_5',\n",
       " 'RFA_6',\n",
       " 'RFA_7',\n",
       " 'RFA_8',\n",
       " 'RFA_9',\n",
       " 'RFA_10',\n",
       " 'RFA_11',\n",
       " 'RFA_12',\n",
       " 'RFA_13',\n",
       " 'RFA_14',\n",
       " 'RFA_15',\n",
       " 'RFA_16',\n",
       " 'RFA_17',\n",
       " 'RFA_18',\n",
       " 'RFA_19',\n",
       " 'RFA_20',\n",
       " 'RFA_21',\n",
       " 'RFA_22',\n",
       " 'RFA_23',\n",
       " 'RFA_24',\n",
       " 'CARDPROM',\n",
       " 'MAXADATE',\n",
       " 'NUMPROM',\n",
       " 'CARDPM12',\n",
       " 'NUMPRM12',\n",
       " 'RAMNTALL',\n",
       " 'NGIFTALL',\n",
       " 'CARDGIFT',\n",
       " 'MINRAMNT',\n",
       " 'MINRDATE',\n",
       " 'MAXRAMNT',\n",
       " 'MAXRDATE',\n",
       " 'LASTGIFT',\n",
       " 'LASTDATE',\n",
       " 'FISTDATE',\n",
       " 'AVGGIFT',\n",
       " 'CONTROLN',\n",
       " 'TARGET_B',\n",
       " 'TARGET_D',\n",
       " 'HPHONE_D',\n",
       " 'RFA_2R',\n",
       " 'RFA_2F',\n",
       " 'RFA_2A',\n",
       " 'MDMAUD_R',\n",
       " 'MDMAUD_F',\n",
       " 'MDMAUD_A',\n",
       " 'CLUSTER2',\n",
       " 'GEOCODE2']"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#This are the columns I want to keep\n",
    "less_ten_10_percent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "9f85cc36",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "400"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Checking how many columns there are left\n",
    "len(less_ten_10_percent)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "cbf9a227",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Putting the data back in a dataframe\n",
    "data = file[less_ten_10_percent]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "9fe48d48",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ODATEDW</th>\n",
       "      <th>OSOURCE</th>\n",
       "      <th>TCODE</th>\n",
       "      <th>STATE</th>\n",
       "      <th>ZIP</th>\n",
       "      <th>MAILCODE</th>\n",
       "      <th>PVASTATE</th>\n",
       "      <th>DOB</th>\n",
       "      <th>NOEXCH</th>\n",
       "      <th>RECINHSE</th>\n",
       "      <th>...</th>\n",
       "      <th>TARGET_D</th>\n",
       "      <th>HPHONE_D</th>\n",
       "      <th>RFA_2R</th>\n",
       "      <th>RFA_2F</th>\n",
       "      <th>RFA_2A</th>\n",
       "      <th>MDMAUD_R</th>\n",
       "      <th>MDMAUD_F</th>\n",
       "      <th>MDMAUD_A</th>\n",
       "      <th>CLUSTER2</th>\n",
       "      <th>GEOCODE2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8901</td>\n",
       "      <td>GRI</td>\n",
       "      <td>0</td>\n",
       "      <td>IL</td>\n",
       "      <td>61081</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>3712</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>L</td>\n",
       "      <td>4</td>\n",
       "      <td>E</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>39.0</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9401</td>\n",
       "      <td>BOA</td>\n",
       "      <td>1</td>\n",
       "      <td>CA</td>\n",
       "      <td>91326</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>5202</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>L</td>\n",
       "      <td>2</td>\n",
       "      <td>G</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>1.0</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9001</td>\n",
       "      <td>AMH</td>\n",
       "      <td>1</td>\n",
       "      <td>NC</td>\n",
       "      <td>27017</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>L</td>\n",
       "      <td>4</td>\n",
       "      <td>E</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>60.0</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8701</td>\n",
       "      <td>BRY</td>\n",
       "      <td>0</td>\n",
       "      <td>CA</td>\n",
       "      <td>95953</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>2801</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>L</td>\n",
       "      <td>4</td>\n",
       "      <td>E</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>41.0</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>8601</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>FL</td>\n",
       "      <td>33176</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>2001</td>\n",
       "      <td>0</td>\n",
       "      <td>X</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>L</td>\n",
       "      <td>2</td>\n",
       "      <td>F</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>26.0</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95407</th>\n",
       "      <td>9601</td>\n",
       "      <td>ASE</td>\n",
       "      <td>1</td>\n",
       "      <td>AK</td>\n",
       "      <td>99504</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>L</td>\n",
       "      <td>1</td>\n",
       "      <td>G</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>12.0</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95408</th>\n",
       "      <td>9601</td>\n",
       "      <td>DCD</td>\n",
       "      <td>1</td>\n",
       "      <td>TX</td>\n",
       "      <td>77379</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>5001</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>L</td>\n",
       "      <td>1</td>\n",
       "      <td>F</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>2.0</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95409</th>\n",
       "      <td>9501</td>\n",
       "      <td>MBC</td>\n",
       "      <td>1</td>\n",
       "      <td>MI</td>\n",
       "      <td>48910</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>3801</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>L</td>\n",
       "      <td>3</td>\n",
       "      <td>E</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>34.0</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95410</th>\n",
       "      <td>8601</td>\n",
       "      <td>PRV</td>\n",
       "      <td>0</td>\n",
       "      <td>CA</td>\n",
       "      <td>91320</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>4005</td>\n",
       "      <td>0</td>\n",
       "      <td>X</td>\n",
       "      <td>...</td>\n",
       "      <td>18.0</td>\n",
       "      <td>1</td>\n",
       "      <td>L</td>\n",
       "      <td>4</td>\n",
       "      <td>F</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>11.0</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95411</th>\n",
       "      <td>8801</td>\n",
       "      <td>MCC</td>\n",
       "      <td>2</td>\n",
       "      <td>NC</td>\n",
       "      <td>28409</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>1801</td>\n",
       "      <td>0</td>\n",
       "      <td>X</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>L</td>\n",
       "      <td>1</td>\n",
       "      <td>G</td>\n",
       "      <td>C</td>\n",
       "      <td>1</td>\n",
       "      <td>C</td>\n",
       "      <td>12.0</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>95412 rows × 400 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       ODATEDW OSOURCE  TCODE STATE    ZIP MAILCODE PVASTATE   DOB NOEXCH  \\\n",
       "0         8901     GRI      0    IL  61081                    3712      0   \n",
       "1         9401     BOA      1    CA  91326                    5202      0   \n",
       "2         9001     AMH      1    NC  27017                       0      0   \n",
       "3         8701     BRY      0    CA  95953                    2801      0   \n",
       "4         8601              0    FL  33176                    2001      0   \n",
       "...        ...     ...    ...   ...    ...      ...      ...   ...    ...   \n",
       "95407     9601     ASE      1    AK  99504                       0      0   \n",
       "95408     9601     DCD      1    TX  77379                    5001      0   \n",
       "95409     9501     MBC      1    MI  48910                    3801      0   \n",
       "95410     8601     PRV      0    CA  91320                    4005      0   \n",
       "95411     8801     MCC      2    NC  28409                    1801      0   \n",
       "\n",
       "      RECINHSE  ... TARGET_D HPHONE_D RFA_2R RFA_2F RFA_2A MDMAUD_R MDMAUD_F  \\\n",
       "0               ...      0.0        0      L      4      E        X        X   \n",
       "1               ...      0.0        0      L      2      G        X        X   \n",
       "2               ...      0.0        1      L      4      E        X        X   \n",
       "3               ...      0.0        1      L      4      E        X        X   \n",
       "4            X  ...      0.0        1      L      2      F        X        X   \n",
       "...        ...  ...      ...      ...    ...    ...    ...      ...      ...   \n",
       "95407           ...      0.0        0      L      1      G        X        X   \n",
       "95408           ...      0.0        1      L      1      F        X        X   \n",
       "95409           ...      0.0        1      L      3      E        X        X   \n",
       "95410        X  ...     18.0        1      L      4      F        X        X   \n",
       "95411        X  ...      0.0        1      L      1      G        C        1   \n",
       "\n",
       "      MDMAUD_A CLUSTER2 GEOCODE2  \n",
       "0            X     39.0        C  \n",
       "1            X      1.0        A  \n",
       "2            X     60.0        C  \n",
       "3            X     41.0        C  \n",
       "4            X     26.0        A  \n",
       "...        ...      ...      ...  \n",
       "95407        X     12.0        C  \n",
       "95408        X      2.0        A  \n",
       "95409        X     34.0        B  \n",
       "95410        X     11.0        A  \n",
       "95411        C     12.0        C  \n",
       "\n",
       "[95412 rows x 400 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "316bfe88",
   "metadata": {},
   "outputs": [],
   "source": [
    "#I am already defening the X variable\n",
    "X = data.drop(columns = ['TARGET_B','TARGET_D','CONTROLN', 'OSOURCE'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "dd6d9a47",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ODATEDW</th>\n",
       "      <th>TCODE</th>\n",
       "      <th>STATE</th>\n",
       "      <th>ZIP</th>\n",
       "      <th>MAILCODE</th>\n",
       "      <th>PVASTATE</th>\n",
       "      <th>DOB</th>\n",
       "      <th>NOEXCH</th>\n",
       "      <th>RECINHSE</th>\n",
       "      <th>RECP3</th>\n",
       "      <th>...</th>\n",
       "      <th>AVGGIFT</th>\n",
       "      <th>HPHONE_D</th>\n",
       "      <th>RFA_2R</th>\n",
       "      <th>RFA_2F</th>\n",
       "      <th>RFA_2A</th>\n",
       "      <th>MDMAUD_R</th>\n",
       "      <th>MDMAUD_F</th>\n",
       "      <th>MDMAUD_A</th>\n",
       "      <th>CLUSTER2</th>\n",
       "      <th>GEOCODE2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8901</td>\n",
       "      <td>0</td>\n",
       "      <td>IL</td>\n",
       "      <td>61081</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>3712</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>7.741935</td>\n",
       "      <td>0</td>\n",
       "      <td>L</td>\n",
       "      <td>4</td>\n",
       "      <td>E</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>39.0</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9401</td>\n",
       "      <td>1</td>\n",
       "      <td>CA</td>\n",
       "      <td>91326</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>5202</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>15.666667</td>\n",
       "      <td>0</td>\n",
       "      <td>L</td>\n",
       "      <td>2</td>\n",
       "      <td>G</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>1.0</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9001</td>\n",
       "      <td>1</td>\n",
       "      <td>NC</td>\n",
       "      <td>27017</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>7.481481</td>\n",
       "      <td>1</td>\n",
       "      <td>L</td>\n",
       "      <td>4</td>\n",
       "      <td>E</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>60.0</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8701</td>\n",
       "      <td>0</td>\n",
       "      <td>CA</td>\n",
       "      <td>95953</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>2801</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>6.812500</td>\n",
       "      <td>1</td>\n",
       "      <td>L</td>\n",
       "      <td>4</td>\n",
       "      <td>E</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>41.0</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>8601</td>\n",
       "      <td>0</td>\n",
       "      <td>FL</td>\n",
       "      <td>33176</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>2001</td>\n",
       "      <td>0</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>...</td>\n",
       "      <td>6.864865</td>\n",
       "      <td>1</td>\n",
       "      <td>L</td>\n",
       "      <td>2</td>\n",
       "      <td>F</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>26.0</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95407</th>\n",
       "      <td>9601</td>\n",
       "      <td>1</td>\n",
       "      <td>AK</td>\n",
       "      <td>99504</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>25.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>L</td>\n",
       "      <td>1</td>\n",
       "      <td>G</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>12.0</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95408</th>\n",
       "      <td>9601</td>\n",
       "      <td>1</td>\n",
       "      <td>TX</td>\n",
       "      <td>77379</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>5001</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>L</td>\n",
       "      <td>1</td>\n",
       "      <td>F</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>2.0</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95409</th>\n",
       "      <td>9501</td>\n",
       "      <td>1</td>\n",
       "      <td>MI</td>\n",
       "      <td>48910</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>3801</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td>X</td>\n",
       "      <td>...</td>\n",
       "      <td>8.285714</td>\n",
       "      <td>1</td>\n",
       "      <td>L</td>\n",
       "      <td>3</td>\n",
       "      <td>E</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>34.0</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95410</th>\n",
       "      <td>8601</td>\n",
       "      <td>0</td>\n",
       "      <td>CA</td>\n",
       "      <td>91320</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>4005</td>\n",
       "      <td>0</td>\n",
       "      <td>X</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>12.146341</td>\n",
       "      <td>1</td>\n",
       "      <td>L</td>\n",
       "      <td>4</td>\n",
       "      <td>F</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>11.0</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95411</th>\n",
       "      <td>8801</td>\n",
       "      <td>2</td>\n",
       "      <td>NC</td>\n",
       "      <td>28409</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>1801</td>\n",
       "      <td>0</td>\n",
       "      <td>X</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>96.794872</td>\n",
       "      <td>1</td>\n",
       "      <td>L</td>\n",
       "      <td>1</td>\n",
       "      <td>G</td>\n",
       "      <td>C</td>\n",
       "      <td>1</td>\n",
       "      <td>C</td>\n",
       "      <td>12.0</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>95412 rows × 396 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       ODATEDW  TCODE STATE    ZIP MAILCODE PVASTATE   DOB NOEXCH RECINHSE  \\\n",
       "0         8901      0    IL  61081                    3712      0            \n",
       "1         9401      1    CA  91326                    5202      0            \n",
       "2         9001      1    NC  27017                       0      0            \n",
       "3         8701      0    CA  95953                    2801      0            \n",
       "4         8601      0    FL  33176                    2001      0        X   \n",
       "...        ...    ...   ...    ...      ...      ...   ...    ...      ...   \n",
       "95407     9601      1    AK  99504                       0      0            \n",
       "95408     9601      1    TX  77379                    5001      0            \n",
       "95409     9501      1    MI  48910                    3801      0            \n",
       "95410     8601      0    CA  91320                    4005      0        X   \n",
       "95411     8801      2    NC  28409                    1801      0        X   \n",
       "\n",
       "      RECP3  ...    AVGGIFT HPHONE_D RFA_2R RFA_2F RFA_2A MDMAUD_R MDMAUD_F  \\\n",
       "0            ...   7.741935        0      L      4      E        X        X   \n",
       "1            ...  15.666667        0      L      2      G        X        X   \n",
       "2            ...   7.481481        1      L      4      E        X        X   \n",
       "3            ...   6.812500        1      L      4      E        X        X   \n",
       "4         X  ...   6.864865        1      L      2      F        X        X   \n",
       "...     ...  ...        ...      ...    ...    ...    ...      ...      ...   \n",
       "95407        ...  25.000000        0      L      1      G        X        X   \n",
       "95408        ...  20.000000        1      L      1      F        X        X   \n",
       "95409     X  ...   8.285714        1      L      3      E        X        X   \n",
       "95410        ...  12.146341        1      L      4      F        X        X   \n",
       "95411        ...  96.794872        1      L      1      G        C        1   \n",
       "\n",
       "      MDMAUD_A CLUSTER2 GEOCODE2  \n",
       "0            X     39.0        C  \n",
       "1            X      1.0        A  \n",
       "2            X     60.0        C  \n",
       "3            X     41.0        C  \n",
       "4            X     26.0        A  \n",
       "...        ...      ...      ...  \n",
       "95407        X     12.0        C  \n",
       "95408        X      2.0        A  \n",
       "95409        X     34.0        B  \n",
       "95410        X     11.0        A  \n",
       "95411        C     12.0        C  \n",
       "\n",
       "[95412 rows x 396 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "f178e34f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#I am splitting the numerical data from the categorical data \n",
    "xnum = X.select_dtypes(include= np.number)\n",
    "xcat = X.select_dtypes(include='object')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "fd9bb0f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "#I am filling the nulls \n",
    "xnum = xnum.fillna(0)\n",
    "xcat = xcat.fillna('')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "cf0519db",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(95412, 73)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xcat.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "0eedd148",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 STATE\n",
      "2 ZIP\n",
      "STATE 0.9958545269431777\n",
      "3 MAILCODE\n",
      "4 PVASTATE\n",
      "STATE 0.8712387319713483\n",
      "5 NOEXCH\n",
      "MAILCODE 0.9610197835149104\n",
      "6 RECINHSE\n",
      "7 RECP3\n",
      "8 RECPGVG\n",
      "RECINHSE 0.2466642159207489\n",
      "9 RECSWEEP\n",
      "STATE 0.6009808546810607\n",
      "10 MDMAUD\n",
      "RECINHSE 0.21901127668839243\n",
      "11 DOMAIN\n",
      "12 CLUSTER\n",
      "13 AGEFLAG\n",
      "14 HOMEOWNR\n",
      "15 CHILD03\n",
      "STATE 0.20247480497655432\n",
      "16 CHILD07\n",
      "RECINHSE 0.5022187873299642\n",
      "17 CHILD12\n",
      "RECINHSE 0.2638372430893398\n",
      "18 CHILD18\n",
      "STATE 0.5893286221985807\n",
      "19 GENDER\n",
      "20 DATASRCE\n",
      "STATE 0.251030736726543\n",
      "21 SOLP3\n",
      "22 SOLIH\n",
      "23 MAJOR\n",
      "RECINHSE 0.910358311652111\n",
      "24 GEOCODE\n",
      "25 COLLECT1\n",
      "26 VETERANS\n",
      "27 BIBLE\n",
      "28 CATLG\n",
      "29 HOMEE\n",
      "DOMAIN 0.49143846696979454\n",
      "30 PETS\n",
      "31 CDPLAY\n",
      "STATE 0.2801031645855597\n",
      "32 STEREO\n",
      "MAILCODE 0.8594281715830145\n",
      "33 PCOWNERS\n",
      "STATE 0.9434806697379569\n",
      "34 PHOTO\n",
      "35 CRAFTS\n",
      "36 FISHER\n",
      "STATE 0.7968480566406873\n",
      "37 GARDENIN\n",
      "38 BOATS\n",
      "STATE 0.3282331689225323\n",
      "39 WALKER\n",
      "40 KIDSTUFF\n",
      "RECINHSE 0.6787292545054087\n",
      "41 CARDS\n",
      "AGEFLAG 0.27679759764810186\n",
      "42 PLATES\n",
      "AGEFLAG 0.24080332333106783\n",
      "43 LIFESRC\n",
      "44 PEPSTRFL\n",
      "45 RFA_2\n",
      "46 RFA_3\n",
      "47 RFA_4\n",
      "48 RFA_5\n",
      "49 RFA_6\n",
      "50 RFA_7\n",
      "51 RFA_8\n",
      "52 RFA_9\n",
      "53 RFA_10\n",
      "54 RFA_11\n",
      "55 RFA_12\n",
      "56 RFA_13\n",
      "57 RFA_14\n",
      "58 RFA_15\n",
      "59 RFA_16\n",
      "60 RFA_17\n",
      "61 RFA_18\n",
      "62 RFA_19\n",
      "63 RFA_20\n",
      "64 RFA_21\n",
      "65 RFA_22\n",
      "66 RFA_23\n",
      "67 RFA_24\n",
      "68 RFA_2R\n",
      "69 RFA_2A\n",
      "70 MDMAUD_R\n",
      "RECINHSE 0.6442811054537825\n",
      "71 MDMAUD_F\n",
      "RECINHSE 0.7579318790458008\n",
      "72 MDMAUD_A\n",
      "RECINHSE 0.7701511577225337\n",
      "73 GEOCODE2\n",
      "STATE 0.1645488217626529\n"
     ]
    }
   ],
   "source": [
    "#Here I am checking the correlation with the xcat, to drop the columns with low correlation \n",
    "cols_to_remove = []\n",
    "\n",
    "i = 0\n",
    "for col1 in xcat.columns:\n",
    "    i += 1\n",
    "    print(i, col1)\n",
    "    for col2 in xcat.columns:\n",
    "        if col1 != col2  and col1 not in cols_to_remove and col2 not in cols_to_remove:\n",
    "            data_crosstab = pd.crosstab(data[col1], data[col2], margins=True)\n",
    "            _, p_value, _, _ = chi2_contingency(data_crosstab)\n",
    "\n",
    "            if p_value <= .01:\n",
    "                data_crosstab = pd.crosstab(data[col1], data['TARGET_B'])\n",
    "                _, p_value1, _, _ = chi2_contingency(data_crosstab)\n",
    "                data_crosstab = pd.crosstab(data[col2], data['TARGET_B'])\n",
    "                _, p_value2, _, _ = chi2_contingency(data_crosstab)\n",
    "\n",
    "                if p_value1 > p_value2 and p_value1 > .1:\n",
    "                    cols_to_remove.append(col1) #name\n",
    "                    print(col2, p_value1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "5bc61a2d",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "25"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Checking how many columns there are to remove\n",
    "len(cols_to_remove)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "04b739da",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(95412, 73)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#chechking the columns, in xcat, they are 74\n",
    "xcat.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "0099f0b0",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Removing the columns from the columns from the x_cat\n",
    "x_cat = xcat.drop(columns = ['ZIP','PVASTATE','NOEXCH','RECPGVG','RECSWEEP','MDMAUD','CHILD03','CHILD07','CHILD12','CHILD18','DATASRCE','MAJOR','HOMEE','CDPLAY','STEREO','PCOWNERS','FISHER','BOATS','KIDSTUFF','CARDS','PLATES','MDMAUD_R','MDMAUD_F','MDMAUD_A','GEOCODE2'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "42d526ea",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>STATE</th>\n",
       "      <th>MAILCODE</th>\n",
       "      <th>RECINHSE</th>\n",
       "      <th>RECP3</th>\n",
       "      <th>DOMAIN</th>\n",
       "      <th>CLUSTER</th>\n",
       "      <th>AGEFLAG</th>\n",
       "      <th>HOMEOWNR</th>\n",
       "      <th>GENDER</th>\n",
       "      <th>SOLP3</th>\n",
       "      <th>...</th>\n",
       "      <th>RFA_17</th>\n",
       "      <th>RFA_18</th>\n",
       "      <th>RFA_19</th>\n",
       "      <th>RFA_20</th>\n",
       "      <th>RFA_21</th>\n",
       "      <th>RFA_22</th>\n",
       "      <th>RFA_23</th>\n",
       "      <th>RFA_24</th>\n",
       "      <th>RFA_2R</th>\n",
       "      <th>RFA_2A</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>IL</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>T2</td>\n",
       "      <td>36</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>F</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>S4E</td>\n",
       "      <td>S4E</td>\n",
       "      <td>S4E</td>\n",
       "      <td>S4E</td>\n",
       "      <td>S4E</td>\n",
       "      <td>S4E</td>\n",
       "      <td>S4E</td>\n",
       "      <td>S4E</td>\n",
       "      <td>L</td>\n",
       "      <td>E</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>CA</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>S1</td>\n",
       "      <td>14</td>\n",
       "      <td>E</td>\n",
       "      <td>H</td>\n",
       "      <td>M</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>N1E</td>\n",
       "      <td>N1E</td>\n",
       "      <td>N1E</td>\n",
       "      <td>N1E</td>\n",
       "      <td></td>\n",
       "      <td>F1E</td>\n",
       "      <td>L</td>\n",
       "      <td>G</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NC</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>R2</td>\n",
       "      <td>43</td>\n",
       "      <td></td>\n",
       "      <td>U</td>\n",
       "      <td>M</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td></td>\n",
       "      <td>S4D</td>\n",
       "      <td>S4D</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>S4D</td>\n",
       "      <td>S4D</td>\n",
       "      <td>S3D</td>\n",
       "      <td>L</td>\n",
       "      <td>E</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>CA</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>R2</td>\n",
       "      <td>44</td>\n",
       "      <td>E</td>\n",
       "      <td>U</td>\n",
       "      <td>F</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>S2D</td>\n",
       "      <td>S2D</td>\n",
       "      <td>A1D</td>\n",
       "      <td>A1D</td>\n",
       "      <td>A1D</td>\n",
       "      <td>A1D</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>L</td>\n",
       "      <td>E</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>FL</td>\n",
       "      <td></td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>S2</td>\n",
       "      <td>16</td>\n",
       "      <td>E</td>\n",
       "      <td>H</td>\n",
       "      <td>F</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>A2D</td>\n",
       "      <td>A2D</td>\n",
       "      <td>A3D</td>\n",
       "      <td>A3D</td>\n",
       "      <td>A3D</td>\n",
       "      <td>I4E</td>\n",
       "      <td>A3D</td>\n",
       "      <td>A3D</td>\n",
       "      <td>L</td>\n",
       "      <td>F</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95407</th>\n",
       "      <td>AK</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>C2</td>\n",
       "      <td>27</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>M</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>L</td>\n",
       "      <td>G</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95408</th>\n",
       "      <td>TX</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>C1</td>\n",
       "      <td>24</td>\n",
       "      <td>E</td>\n",
       "      <td>H</td>\n",
       "      <td>M</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>L</td>\n",
       "      <td>F</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95409</th>\n",
       "      <td>MI</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>X</td>\n",
       "      <td>C3</td>\n",
       "      <td>30</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>M</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td></td>\n",
       "      <td>F1D</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>P1D</td>\n",
       "      <td>P1D</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>L</td>\n",
       "      <td>E</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95410</th>\n",
       "      <td>CA</td>\n",
       "      <td></td>\n",
       "      <td>X</td>\n",
       "      <td></td>\n",
       "      <td>C1</td>\n",
       "      <td>24</td>\n",
       "      <td>E</td>\n",
       "      <td>H</td>\n",
       "      <td>F</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>S2F</td>\n",
       "      <td>S2F</td>\n",
       "      <td>A1F</td>\n",
       "      <td>A1F</td>\n",
       "      <td>A1F</td>\n",
       "      <td>A1F</td>\n",
       "      <td>S2F</td>\n",
       "      <td>S3F</td>\n",
       "      <td>L</td>\n",
       "      <td>F</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95411</th>\n",
       "      <td>NC</td>\n",
       "      <td></td>\n",
       "      <td>X</td>\n",
       "      <td></td>\n",
       "      <td>C1</td>\n",
       "      <td>24</td>\n",
       "      <td>E</td>\n",
       "      <td>U</td>\n",
       "      <td>F</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>S3G</td>\n",
       "      <td>S3G</td>\n",
       "      <td>S4G</td>\n",
       "      <td>S3G</td>\n",
       "      <td>S3G</td>\n",
       "      <td>S3G</td>\n",
       "      <td>S3G</td>\n",
       "      <td>S3G</td>\n",
       "      <td>L</td>\n",
       "      <td>G</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>95412 rows × 48 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      STATE MAILCODE RECINHSE RECP3 DOMAIN CLUSTER AGEFLAG HOMEOWNR GENDER  \\\n",
       "0        IL                             T2      36                       F   \n",
       "1        CA                             S1      14       E        H      M   \n",
       "2        NC                             R2      43                U      M   \n",
       "3        CA                             R2      44       E        U      F   \n",
       "4        FL                 X     X     S2      16       E        H      F   \n",
       "...     ...      ...      ...   ...    ...     ...     ...      ...    ...   \n",
       "95407    AK                             C2      27                       M   \n",
       "95408    TX                             C1      24       E        H      M   \n",
       "95409    MI                       X     C3      30                       M   \n",
       "95410    CA                 X           C1      24       E        H      F   \n",
       "95411    NC                 X           C1      24       E        U      F   \n",
       "\n",
       "      SOLP3  ... RFA_17 RFA_18 RFA_19 RFA_20 RFA_21 RFA_22 RFA_23 RFA_24  \\\n",
       "0            ...    S4E    S4E    S4E    S4E    S4E    S4E    S4E    S4E   \n",
       "1            ...                  N1E    N1E    N1E    N1E           F1E   \n",
       "2            ...           S4D    S4D                  S4D    S4D    S3D   \n",
       "3            ...    S2D    S2D    A1D    A1D    A1D    A1D                 \n",
       "4            ...    A2D    A2D    A3D    A3D    A3D    I4E    A3D    A3D   \n",
       "...     ...  ...    ...    ...    ...    ...    ...    ...    ...    ...   \n",
       "95407        ...                                                           \n",
       "95408        ...                                                           \n",
       "95409        ...           F1D                  P1D    P1D                 \n",
       "95410        ...    S2F    S2F    A1F    A1F    A1F    A1F    S2F    S3F   \n",
       "95411        ...    S3G    S3G    S4G    S3G    S3G    S3G    S3G    S3G   \n",
       "\n",
       "      RFA_2R RFA_2A  \n",
       "0          L      E  \n",
       "1          L      G  \n",
       "2          L      E  \n",
       "3          L      E  \n",
       "4          L      F  \n",
       "...      ...    ...  \n",
       "95407      L      G  \n",
       "95408      L      F  \n",
       "95409      L      E  \n",
       "95410      L      F  \n",
       "95411      L      G  \n",
       "\n",
       "[95412 rows x 48 columns]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_cat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "701c65c3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RFA_6\n",
      "RFA_7\n",
      "RFA_8\n",
      "RFA_9\n",
      "RFA_11\n",
      "RFA_12\n",
      "RFA_16\n",
      "RFA_17\n",
      "RFA_18\n",
      "RFA_19\n",
      "RFA_21\n",
      "RFA_22\n"
     ]
    },
    {
     "ename": "KeyError",
     "evalue": "'OSOURCE'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m~/opt/anaconda3/lib/python3.8/site-packages/pandas/core/indexes/base.py\u001b[0m in \u001b[0;36mget_loc\u001b[0;34m(self, key, method, tolerance)\u001b[0m\n\u001b[1;32m   3079\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3080\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcasted_key\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3081\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0merr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: 'OSOURCE'",
      "\nThe above exception was the direct cause of the following exception:\n",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-32-aa272ba0a9b0>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx_cat\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mc\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munique\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m100\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx_cat\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'OSOURCE'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munique\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/opt/anaconda3/lib/python3.8/site-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m   3022\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnlevels\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3023\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_getitem_multilevel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3024\u001b[0;31m             \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3025\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mis_integer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindexer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3026\u001b[0m                 \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mindexer\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/opt/anaconda3/lib/python3.8/site-packages/pandas/core/indexes/base.py\u001b[0m in \u001b[0;36mget_loc\u001b[0;34m(self, key, method, tolerance)\u001b[0m\n\u001b[1;32m   3080\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcasted_key\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3081\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0merr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3082\u001b[0;31m                 \u001b[0;32mraise\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0merr\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3083\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3084\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mtolerance\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: 'OSOURCE'"
     ]
    }
   ],
   "source": [
    "#checking the columns, in x_cat, the are 49 (74-25 = 49)\n",
    "x_cat.shape\n",
    "\n",
    "for c in x_cat:\n",
    "    if len(x_cat[c].unique()) > 100:\n",
    "        print(c)\n",
    "len(x_cat['OSOURCE'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "68e2dc72",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(95412, 67)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#The numerical data:\n",
    "#scaling the data\n",
    "#Using PCA to put the columns in a smaller set of columns \n",
    "\n",
    "scaler = MinMaxScaler()\n",
    "x_scaled = scaler.fit_transform(xnum)\n",
    "\n",
    "pca = PCA(n_components=.95)\n",
    "pca.fit(x_scaled, data['TARGET_B'])\n",
    "xnum = pca.transform(xnum)\n",
    "\n",
    "xnum.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "e0f0c89e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>STATE</th>\n",
       "      <th>MAILCODE</th>\n",
       "      <th>RECINHSE</th>\n",
       "      <th>RECP3</th>\n",
       "      <th>DOMAIN</th>\n",
       "      <th>CLUSTER</th>\n",
       "      <th>AGEFLAG</th>\n",
       "      <th>HOMEOWNR</th>\n",
       "      <th>GENDER</th>\n",
       "      <th>SOLP3</th>\n",
       "      <th>...</th>\n",
       "      <th>RFA_17</th>\n",
       "      <th>RFA_18</th>\n",
       "      <th>RFA_19</th>\n",
       "      <th>RFA_20</th>\n",
       "      <th>RFA_21</th>\n",
       "      <th>RFA_22</th>\n",
       "      <th>RFA_23</th>\n",
       "      <th>RFA_24</th>\n",
       "      <th>RFA_2R</th>\n",
       "      <th>RFA_2A</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>IL</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>T2</td>\n",
       "      <td>36</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>F</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>S4E</td>\n",
       "      <td>S4E</td>\n",
       "      <td>S4E</td>\n",
       "      <td>S4E</td>\n",
       "      <td>S4E</td>\n",
       "      <td>S4E</td>\n",
       "      <td>S4E</td>\n",
       "      <td>S4E</td>\n",
       "      <td>L</td>\n",
       "      <td>E</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>CA</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>S1</td>\n",
       "      <td>14</td>\n",
       "      <td>E</td>\n",
       "      <td>H</td>\n",
       "      <td>M</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>N1E</td>\n",
       "      <td>N1E</td>\n",
       "      <td>N1E</td>\n",
       "      <td>N1E</td>\n",
       "      <td></td>\n",
       "      <td>F1E</td>\n",
       "      <td>L</td>\n",
       "      <td>G</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NC</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>R2</td>\n",
       "      <td>43</td>\n",
       "      <td></td>\n",
       "      <td>U</td>\n",
       "      <td>M</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td></td>\n",
       "      <td>S4D</td>\n",
       "      <td>S4D</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>S4D</td>\n",
       "      <td>S4D</td>\n",
       "      <td>S3D</td>\n",
       "      <td>L</td>\n",
       "      <td>E</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>CA</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>R2</td>\n",
       "      <td>44</td>\n",
       "      <td>E</td>\n",
       "      <td>U</td>\n",
       "      <td>F</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>S2D</td>\n",
       "      <td>S2D</td>\n",
       "      <td>A1D</td>\n",
       "      <td>A1D</td>\n",
       "      <td>A1D</td>\n",
       "      <td>A1D</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>L</td>\n",
       "      <td>E</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>FL</td>\n",
       "      <td></td>\n",
       "      <td>X</td>\n",
       "      <td>X</td>\n",
       "      <td>S2</td>\n",
       "      <td>16</td>\n",
       "      <td>E</td>\n",
       "      <td>H</td>\n",
       "      <td>F</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>A2D</td>\n",
       "      <td>A2D</td>\n",
       "      <td>A3D</td>\n",
       "      <td>A3D</td>\n",
       "      <td>A3D</td>\n",
       "      <td>I4E</td>\n",
       "      <td>A3D</td>\n",
       "      <td>A3D</td>\n",
       "      <td>L</td>\n",
       "      <td>F</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95407</th>\n",
       "      <td>AK</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>C2</td>\n",
       "      <td>27</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>M</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>L</td>\n",
       "      <td>G</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95408</th>\n",
       "      <td>TX</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>C1</td>\n",
       "      <td>24</td>\n",
       "      <td>E</td>\n",
       "      <td>H</td>\n",
       "      <td>M</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>L</td>\n",
       "      <td>F</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95409</th>\n",
       "      <td>MI</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>X</td>\n",
       "      <td>C3</td>\n",
       "      <td>30</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>M</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td></td>\n",
       "      <td>F1D</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>P1D</td>\n",
       "      <td>P1D</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>L</td>\n",
       "      <td>E</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95410</th>\n",
       "      <td>CA</td>\n",
       "      <td></td>\n",
       "      <td>X</td>\n",
       "      <td></td>\n",
       "      <td>C1</td>\n",
       "      <td>24</td>\n",
       "      <td>E</td>\n",
       "      <td>H</td>\n",
       "      <td>F</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>S2F</td>\n",
       "      <td>S2F</td>\n",
       "      <td>A1F</td>\n",
       "      <td>A1F</td>\n",
       "      <td>A1F</td>\n",
       "      <td>A1F</td>\n",
       "      <td>S2F</td>\n",
       "      <td>S3F</td>\n",
       "      <td>L</td>\n",
       "      <td>F</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95411</th>\n",
       "      <td>NC</td>\n",
       "      <td></td>\n",
       "      <td>X</td>\n",
       "      <td></td>\n",
       "      <td>C1</td>\n",
       "      <td>24</td>\n",
       "      <td>E</td>\n",
       "      <td>U</td>\n",
       "      <td>F</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>S3G</td>\n",
       "      <td>S3G</td>\n",
       "      <td>S4G</td>\n",
       "      <td>S3G</td>\n",
       "      <td>S3G</td>\n",
       "      <td>S3G</td>\n",
       "      <td>S3G</td>\n",
       "      <td>S3G</td>\n",
       "      <td>L</td>\n",
       "      <td>G</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>95412 rows × 48 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      STATE MAILCODE RECINHSE RECP3 DOMAIN CLUSTER AGEFLAG HOMEOWNR GENDER  \\\n",
       "0        IL                             T2      36                       F   \n",
       "1        CA                             S1      14       E        H      M   \n",
       "2        NC                             R2      43                U      M   \n",
       "3        CA                             R2      44       E        U      F   \n",
       "4        FL                 X     X     S2      16       E        H      F   \n",
       "...     ...      ...      ...   ...    ...     ...     ...      ...    ...   \n",
       "95407    AK                             C2      27                       M   \n",
       "95408    TX                             C1      24       E        H      M   \n",
       "95409    MI                       X     C3      30                       M   \n",
       "95410    CA                 X           C1      24       E        H      F   \n",
       "95411    NC                 X           C1      24       E        U      F   \n",
       "\n",
       "      SOLP3  ... RFA_17 RFA_18 RFA_19 RFA_20 RFA_21 RFA_22 RFA_23 RFA_24  \\\n",
       "0            ...    S4E    S4E    S4E    S4E    S4E    S4E    S4E    S4E   \n",
       "1            ...                  N1E    N1E    N1E    N1E           F1E   \n",
       "2            ...           S4D    S4D                  S4D    S4D    S3D   \n",
       "3            ...    S2D    S2D    A1D    A1D    A1D    A1D                 \n",
       "4            ...    A2D    A2D    A3D    A3D    A3D    I4E    A3D    A3D   \n",
       "...     ...  ...    ...    ...    ...    ...    ...    ...    ...    ...   \n",
       "95407        ...                                                           \n",
       "95408        ...                                                           \n",
       "95409        ...           F1D                  P1D    P1D                 \n",
       "95410        ...    S2F    S2F    A1F    A1F    A1F    A1F    S2F    S3F   \n",
       "95411        ...    S3G    S3G    S4G    S3G    S3G    S3G    S3G    S3G   \n",
       "\n",
       "      RFA_2R RFA_2A  \n",
       "0          L      E  \n",
       "1          L      G  \n",
       "2          L      E  \n",
       "3          L      E  \n",
       "4          L      F  \n",
       "...      ...    ...  \n",
       "95407      L      G  \n",
       "95408      L      F  \n",
       "95409      L      E  \n",
       "95410      L      F  \n",
       "95411      L      G  \n",
       "\n",
       "[95412 rows x 48 columns]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# I want to concate x_cat and xnum, I want to see how they look like separate\n",
    "x_cat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "1bcbfcff",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(95412, 2290)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dummy_cat = pd.get_dummies(x_cat)\n",
    "dummy_cat.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "14ddcfc6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(95412, 540)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scaler = MinMaxScaler()\n",
    "x_scaled_cat = scaler.fit_transform(dummy_cat)\n",
    "\n",
    "pca = PCA(n_components=.95)\n",
    "pca.fit(x_scaled_cat, data['TARGET_B'])\n",
    "dummy_cat = pca.transform(x_scaled_cat)\n",
    "dummy_cat.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "065a27d6",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[  831.88322727,  -646.8618953 ,  -363.95531641, ...,\n",
       "          676.66063552,  2064.07174067,  -632.2305592 ],\n",
       "       [ 3574.42415852, -1134.82991987,  -210.07112523, ...,\n",
       "         1459.62266889,  2928.57942188,   137.53503803],\n",
       "       [  681.97323327,  -547.51104097,   -62.10406478, ...,\n",
       "         1677.90377228,  2317.46486733,   202.28531008],\n",
       "       ...,\n",
       "       [ 1368.55096204,  -367.42454848,  -195.87731252, ...,\n",
       "          596.46384559,  2328.66613834,  -871.68894553],\n",
       "       [ 2770.20504128,  -537.11849453,   -92.37382704, ...,\n",
       "          792.33146357,  1922.31697309,  -339.82300727],\n",
       "       [ 2407.24142578,  -449.2485689 ,   114.89424986, ...,\n",
       "         1206.67970445,  2230.77455508,  -104.75087664]])"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xnum"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "c3a37a68",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Here I am making a dataframe of xnum, so I can concate it with with x_cat\n",
    "df = pd.DataFrame(xnum)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "c0b0096a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0     float64\n",
       "1     float64\n",
       "2     float64\n",
       "3     float64\n",
       "4     float64\n",
       "       ...   \n",
       "62    float64\n",
       "63    float64\n",
       "64    float64\n",
       "65    float64\n",
       "66    float64\n",
       "Length: 67, dtype: object"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "304f6748",
   "metadata": {},
   "outputs": [],
   "source": [
    "dummy_cat = pd.DataFrame(dummy_cat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "e0939eb2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>530</th>\n",
       "      <th>531</th>\n",
       "      <th>532</th>\n",
       "      <th>533</th>\n",
       "      <th>534</th>\n",
       "      <th>535</th>\n",
       "      <th>536</th>\n",
       "      <th>537</th>\n",
       "      <th>538</th>\n",
       "      <th>539</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-1.543898</td>\n",
       "      <td>-1.414390</td>\n",
       "      <td>-1.182505</td>\n",
       "      <td>-0.913462</td>\n",
       "      <td>-1.019044</td>\n",
       "      <td>-0.186667</td>\n",
       "      <td>0.458338</td>\n",
       "      <td>0.442526</td>\n",
       "      <td>0.042676</td>\n",
       "      <td>-0.057756</td>\n",
       "      <td>...</td>\n",
       "      <td>0.055177</td>\n",
       "      <td>0.058121</td>\n",
       "      <td>0.016822</td>\n",
       "      <td>-0.056233</td>\n",
       "      <td>-0.055087</td>\n",
       "      <td>0.007557</td>\n",
       "      <td>-0.008825</td>\n",
       "      <td>-0.036627</td>\n",
       "      <td>0.016017</td>\n",
       "      <td>0.082159</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.118673</td>\n",
       "      <td>-0.286203</td>\n",
       "      <td>0.037046</td>\n",
       "      <td>0.394775</td>\n",
       "      <td>1.994977</td>\n",
       "      <td>0.372205</td>\n",
       "      <td>-0.605139</td>\n",
       "      <td>-0.683514</td>\n",
       "      <td>0.107954</td>\n",
       "      <td>0.061136</td>\n",
       "      <td>...</td>\n",
       "      <td>0.020850</td>\n",
       "      <td>0.008793</td>\n",
       "      <td>0.002238</td>\n",
       "      <td>-0.010681</td>\n",
       "      <td>0.049743</td>\n",
       "      <td>0.065684</td>\n",
       "      <td>-0.025363</td>\n",
       "      <td>-0.030930</td>\n",
       "      <td>-0.004446</td>\n",
       "      <td>-0.015777</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.582553</td>\n",
       "      <td>-1.445814</td>\n",
       "      <td>-0.959750</td>\n",
       "      <td>-0.740996</td>\n",
       "      <td>-1.104002</td>\n",
       "      <td>-0.566962</td>\n",
       "      <td>0.356428</td>\n",
       "      <td>-0.978498</td>\n",
       "      <td>-0.136501</td>\n",
       "      <td>0.092563</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.135746</td>\n",
       "      <td>0.069506</td>\n",
       "      <td>-0.141231</td>\n",
       "      <td>-0.040676</td>\n",
       "      <td>0.057974</td>\n",
       "      <td>0.053992</td>\n",
       "      <td>0.123118</td>\n",
       "      <td>0.051522</td>\n",
       "      <td>-0.030138</td>\n",
       "      <td>0.116640</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.908226</td>\n",
       "      <td>-1.223641</td>\n",
       "      <td>-0.441972</td>\n",
       "      <td>-0.822089</td>\n",
       "      <td>-0.632321</td>\n",
       "      <td>0.125704</td>\n",
       "      <td>-0.683564</td>\n",
       "      <td>0.346862</td>\n",
       "      <td>0.168112</td>\n",
       "      <td>-0.333127</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.018941</td>\n",
       "      <td>-0.025626</td>\n",
       "      <td>-0.028587</td>\n",
       "      <td>-0.018650</td>\n",
       "      <td>0.068444</td>\n",
       "      <td>-0.033556</td>\n",
       "      <td>0.028733</td>\n",
       "      <td>0.031233</td>\n",
       "      <td>-0.019987</td>\n",
       "      <td>-0.081835</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.893818</td>\n",
       "      <td>0.359498</td>\n",
       "      <td>1.994094</td>\n",
       "      <td>-0.306228</td>\n",
       "      <td>0.522387</td>\n",
       "      <td>1.086882</td>\n",
       "      <td>0.608042</td>\n",
       "      <td>0.736720</td>\n",
       "      <td>-0.539454</td>\n",
       "      <td>1.112968</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.047736</td>\n",
       "      <td>-0.013447</td>\n",
       "      <td>0.021439</td>\n",
       "      <td>0.078648</td>\n",
       "      <td>0.003598</td>\n",
       "      <td>0.050231</td>\n",
       "      <td>-0.010456</td>\n",
       "      <td>0.005339</td>\n",
       "      <td>0.085180</td>\n",
       "      <td>0.138701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95407</th>\n",
       "      <td>2.812180</td>\n",
       "      <td>-0.992399</td>\n",
       "      <td>-1.024800</td>\n",
       "      <td>0.648204</td>\n",
       "      <td>0.406791</td>\n",
       "      <td>-0.721104</td>\n",
       "      <td>0.749952</td>\n",
       "      <td>-0.554647</td>\n",
       "      <td>0.570987</td>\n",
       "      <td>0.301602</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.016459</td>\n",
       "      <td>-0.006181</td>\n",
       "      <td>0.025597</td>\n",
       "      <td>0.002865</td>\n",
       "      <td>-0.031965</td>\n",
       "      <td>0.010783</td>\n",
       "      <td>-0.007026</td>\n",
       "      <td>-0.029539</td>\n",
       "      <td>0.007391</td>\n",
       "      <td>-0.014890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95408</th>\n",
       "      <td>2.970057</td>\n",
       "      <td>0.118872</td>\n",
       "      <td>-0.022766</td>\n",
       "      <td>-0.071505</td>\n",
       "      <td>0.297184</td>\n",
       "      <td>-0.843509</td>\n",
       "      <td>-0.807151</td>\n",
       "      <td>-0.756627</td>\n",
       "      <td>-0.336047</td>\n",
       "      <td>0.358324</td>\n",
       "      <td>...</td>\n",
       "      <td>0.001280</td>\n",
       "      <td>0.047298</td>\n",
       "      <td>-0.063582</td>\n",
       "      <td>-0.011937</td>\n",
       "      <td>0.017651</td>\n",
       "      <td>0.010413</td>\n",
       "      <td>-0.018632</td>\n",
       "      <td>0.048859</td>\n",
       "      <td>0.009844</td>\n",
       "      <td>0.033587</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95409</th>\n",
       "      <td>-0.048154</td>\n",
       "      <td>-1.252510</td>\n",
       "      <td>-0.998494</td>\n",
       "      <td>-0.501840</td>\n",
       "      <td>-0.747076</td>\n",
       "      <td>0.579754</td>\n",
       "      <td>0.594215</td>\n",
       "      <td>-1.011395</td>\n",
       "      <td>0.679233</td>\n",
       "      <td>-0.419613</td>\n",
       "      <td>...</td>\n",
       "      <td>0.095174</td>\n",
       "      <td>0.197797</td>\n",
       "      <td>0.122209</td>\n",
       "      <td>-0.000735</td>\n",
       "      <td>0.120857</td>\n",
       "      <td>-0.158416</td>\n",
       "      <td>-0.051092</td>\n",
       "      <td>0.001329</td>\n",
       "      <td>-0.231483</td>\n",
       "      <td>0.176777</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95410</th>\n",
       "      <td>-1.582883</td>\n",
       "      <td>-0.147436</td>\n",
       "      <td>0.139371</td>\n",
       "      <td>-0.026801</td>\n",
       "      <td>-0.757344</td>\n",
       "      <td>-0.551932</td>\n",
       "      <td>-0.883893</td>\n",
       "      <td>0.369235</td>\n",
       "      <td>-0.907509</td>\n",
       "      <td>0.809890</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.063948</td>\n",
       "      <td>0.076816</td>\n",
       "      <td>0.026445</td>\n",
       "      <td>-0.034682</td>\n",
       "      <td>-0.050798</td>\n",
       "      <td>0.052023</td>\n",
       "      <td>-0.034453</td>\n",
       "      <td>0.098954</td>\n",
       "      <td>-0.009225</td>\n",
       "      <td>-0.044114</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95411</th>\n",
       "      <td>-1.693127</td>\n",
       "      <td>-1.376564</td>\n",
       "      <td>-0.501880</td>\n",
       "      <td>2.063288</td>\n",
       "      <td>0.342977</td>\n",
       "      <td>-0.620260</td>\n",
       "      <td>-0.538404</td>\n",
       "      <td>0.694080</td>\n",
       "      <td>-0.562500</td>\n",
       "      <td>-0.201557</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.215516</td>\n",
       "      <td>-0.104608</td>\n",
       "      <td>0.003286</td>\n",
       "      <td>-0.224950</td>\n",
       "      <td>0.055244</td>\n",
       "      <td>0.007030</td>\n",
       "      <td>-0.042468</td>\n",
       "      <td>-0.009838</td>\n",
       "      <td>0.018166</td>\n",
       "      <td>-0.009733</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>95412 rows × 540 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            0         1         2         3         4         5         6    \\\n",
       "0     -1.543898 -1.414390 -1.182505 -0.913462 -1.019044 -0.186667  0.458338   \n",
       "1      0.118673 -0.286203  0.037046  0.394775  1.994977  0.372205 -0.605139   \n",
       "2     -0.582553 -1.445814 -0.959750 -0.740996 -1.104002 -0.566962  0.356428   \n",
       "3     -0.908226 -1.223641 -0.441972 -0.822089 -0.632321  0.125704 -0.683564   \n",
       "4     -0.893818  0.359498  1.994094 -0.306228  0.522387  1.086882  0.608042   \n",
       "...         ...       ...       ...       ...       ...       ...       ...   \n",
       "95407  2.812180 -0.992399 -1.024800  0.648204  0.406791 -0.721104  0.749952   \n",
       "95408  2.970057  0.118872 -0.022766 -0.071505  0.297184 -0.843509 -0.807151   \n",
       "95409 -0.048154 -1.252510 -0.998494 -0.501840 -0.747076  0.579754  0.594215   \n",
       "95410 -1.582883 -0.147436  0.139371 -0.026801 -0.757344 -0.551932 -0.883893   \n",
       "95411 -1.693127 -1.376564 -0.501880  2.063288  0.342977 -0.620260 -0.538404   \n",
       "\n",
       "            7         8         9    ...       530       531       532  \\\n",
       "0      0.442526  0.042676 -0.057756  ...  0.055177  0.058121  0.016822   \n",
       "1     -0.683514  0.107954  0.061136  ...  0.020850  0.008793  0.002238   \n",
       "2     -0.978498 -0.136501  0.092563  ... -0.135746  0.069506 -0.141231   \n",
       "3      0.346862  0.168112 -0.333127  ... -0.018941 -0.025626 -0.028587   \n",
       "4      0.736720 -0.539454  1.112968  ... -0.047736 -0.013447  0.021439   \n",
       "...         ...       ...       ...  ...       ...       ...       ...   \n",
       "95407 -0.554647  0.570987  0.301602  ... -0.016459 -0.006181  0.025597   \n",
       "95408 -0.756627 -0.336047  0.358324  ...  0.001280  0.047298 -0.063582   \n",
       "95409 -1.011395  0.679233 -0.419613  ...  0.095174  0.197797  0.122209   \n",
       "95410  0.369235 -0.907509  0.809890  ... -0.063948  0.076816  0.026445   \n",
       "95411  0.694080 -0.562500 -0.201557  ... -0.215516 -0.104608  0.003286   \n",
       "\n",
       "            533       534       535       536       537       538       539  \n",
       "0     -0.056233 -0.055087  0.007557 -0.008825 -0.036627  0.016017  0.082159  \n",
       "1     -0.010681  0.049743  0.065684 -0.025363 -0.030930 -0.004446 -0.015777  \n",
       "2     -0.040676  0.057974  0.053992  0.123118  0.051522 -0.030138  0.116640  \n",
       "3     -0.018650  0.068444 -0.033556  0.028733  0.031233 -0.019987 -0.081835  \n",
       "4      0.078648  0.003598  0.050231 -0.010456  0.005339  0.085180  0.138701  \n",
       "...         ...       ...       ...       ...       ...       ...       ...  \n",
       "95407  0.002865 -0.031965  0.010783 -0.007026 -0.029539  0.007391 -0.014890  \n",
       "95408 -0.011937  0.017651  0.010413 -0.018632  0.048859  0.009844  0.033587  \n",
       "95409 -0.000735  0.120857 -0.158416 -0.051092  0.001329 -0.231483  0.176777  \n",
       "95410 -0.034682 -0.050798  0.052023 -0.034453  0.098954 -0.009225 -0.044114  \n",
       "95411 -0.224950  0.055244  0.007030 -0.042468 -0.009838  0.018166 -0.009733  \n",
       "\n",
       "[95412 rows x 540 columns]"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dummy_cat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "0541c063",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0      float64\n",
       "1      float64\n",
       "2      float64\n",
       "3      float64\n",
       "4      float64\n",
       "        ...   \n",
       "535    float64\n",
       "536    float64\n",
       "537    float64\n",
       "538    float64\n",
       "539    float64\n",
       "Length: 540, dtype: object"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dummy_cat.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "5d6b0e15",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0      float64\n",
       "1      float64\n",
       "2      float64\n",
       "3      float64\n",
       "4      float64\n",
       "        ...   \n",
       "535    float64\n",
       "536    float64\n",
       "537    float64\n",
       "538    float64\n",
       "539    float64\n",
       "Length: 540, dtype: object"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f = dummy_cat.astype(float)\n",
    "f.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "2810aaa6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>530</th>\n",
       "      <th>531</th>\n",
       "      <th>532</th>\n",
       "      <th>533</th>\n",
       "      <th>534</th>\n",
       "      <th>535</th>\n",
       "      <th>536</th>\n",
       "      <th>537</th>\n",
       "      <th>538</th>\n",
       "      <th>539</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>831.883227</td>\n",
       "      <td>-646.861895</td>\n",
       "      <td>-363.955316</td>\n",
       "      <td>-182.229876</td>\n",
       "      <td>-400.367679</td>\n",
       "      <td>-381.638509</td>\n",
       "      <td>265.626351</td>\n",
       "      <td>277.234640</td>\n",
       "      <td>21.686036</td>\n",
       "      <td>3375.632218</td>\n",
       "      <td>...</td>\n",
       "      <td>0.055177</td>\n",
       "      <td>0.058121</td>\n",
       "      <td>0.016822</td>\n",
       "      <td>-0.056233</td>\n",
       "      <td>-0.055087</td>\n",
       "      <td>0.007557</td>\n",
       "      <td>-0.008825</td>\n",
       "      <td>-0.036627</td>\n",
       "      <td>0.016017</td>\n",
       "      <td>0.082159</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3574.424159</td>\n",
       "      <td>-1134.829920</td>\n",
       "      <td>-210.071125</td>\n",
       "      <td>1506.846801</td>\n",
       "      <td>-710.712659</td>\n",
       "      <td>-1000.928500</td>\n",
       "      <td>697.962871</td>\n",
       "      <td>-156.237400</td>\n",
       "      <td>-541.669607</td>\n",
       "      <td>2921.697086</td>\n",
       "      <td>...</td>\n",
       "      <td>0.020850</td>\n",
       "      <td>0.008793</td>\n",
       "      <td>0.002238</td>\n",
       "      <td>-0.010681</td>\n",
       "      <td>0.049743</td>\n",
       "      <td>0.065684</td>\n",
       "      <td>-0.025363</td>\n",
       "      <td>-0.030930</td>\n",
       "      <td>-0.004446</td>\n",
       "      <td>-0.015777</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>681.973233</td>\n",
       "      <td>-547.511041</td>\n",
       "      <td>-62.104065</td>\n",
       "      <td>-152.397865</td>\n",
       "      <td>-532.179443</td>\n",
       "      <td>-663.551389</td>\n",
       "      <td>-36.567874</td>\n",
       "      <td>297.875608</td>\n",
       "      <td>-238.958811</td>\n",
       "      <td>3497.992065</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.135746</td>\n",
       "      <td>0.069506</td>\n",
       "      <td>-0.141231</td>\n",
       "      <td>-0.040676</td>\n",
       "      <td>0.057974</td>\n",
       "      <td>0.053992</td>\n",
       "      <td>0.123118</td>\n",
       "      <td>0.051522</td>\n",
       "      <td>-0.030138</td>\n",
       "      <td>0.116640</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2248.176026</td>\n",
       "      <td>-371.333556</td>\n",
       "      <td>17.341024</td>\n",
       "      <td>153.031358</td>\n",
       "      <td>-559.494031</td>\n",
       "      <td>202.556490</td>\n",
       "      <td>61.571859</td>\n",
       "      <td>-1623.052607</td>\n",
       "      <td>-1222.283285</td>\n",
       "      <td>3686.672549</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.018941</td>\n",
       "      <td>-0.025626</td>\n",
       "      <td>-0.028587</td>\n",
       "      <td>-0.018650</td>\n",
       "      <td>0.068444</td>\n",
       "      <td>-0.033556</td>\n",
       "      <td>0.028733</td>\n",
       "      <td>0.031233</td>\n",
       "      <td>-0.019987</td>\n",
       "      <td>-0.081835</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1424.241454</td>\n",
       "      <td>-326.772925</td>\n",
       "      <td>-135.899676</td>\n",
       "      <td>-236.083293</td>\n",
       "      <td>-504.559563</td>\n",
       "      <td>9.072990</td>\n",
       "      <td>-37.746297</td>\n",
       "      <td>-853.493295</td>\n",
       "      <td>-824.684647</td>\n",
       "      <td>3658.558861</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.047736</td>\n",
       "      <td>-0.013447</td>\n",
       "      <td>0.021439</td>\n",
       "      <td>0.078648</td>\n",
       "      <td>0.003598</td>\n",
       "      <td>0.050231</td>\n",
       "      <td>-0.010456</td>\n",
       "      <td>0.005339</td>\n",
       "      <td>0.085180</td>\n",
       "      <td>0.138701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95407</th>\n",
       "      <td>1466.972344</td>\n",
       "      <td>-537.481623</td>\n",
       "      <td>417.465311</td>\n",
       "      <td>-241.452684</td>\n",
       "      <td>-1178.093742</td>\n",
       "      <td>-1058.511164</td>\n",
       "      <td>-443.431369</td>\n",
       "      <td>560.561072</td>\n",
       "      <td>-127.990495</td>\n",
       "      <td>-1053.981987</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.016459</td>\n",
       "      <td>-0.006181</td>\n",
       "      <td>0.025597</td>\n",
       "      <td>0.002865</td>\n",
       "      <td>-0.031965</td>\n",
       "      <td>0.010783</td>\n",
       "      <td>-0.007026</td>\n",
       "      <td>-0.029539</td>\n",
       "      <td>0.007391</td>\n",
       "      <td>-0.014890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95408</th>\n",
       "      <td>2345.345493</td>\n",
       "      <td>-803.998002</td>\n",
       "      <td>73.513108</td>\n",
       "      <td>406.864479</td>\n",
       "      <td>-728.035887</td>\n",
       "      <td>-55.330810</td>\n",
       "      <td>184.177274</td>\n",
       "      <td>-172.615589</td>\n",
       "      <td>535.689080</td>\n",
       "      <td>-2955.636037</td>\n",
       "      <td>...</td>\n",
       "      <td>0.001280</td>\n",
       "      <td>0.047298</td>\n",
       "      <td>-0.063582</td>\n",
       "      <td>-0.011937</td>\n",
       "      <td>0.017651</td>\n",
       "      <td>0.010413</td>\n",
       "      <td>-0.018632</td>\n",
       "      <td>0.048859</td>\n",
       "      <td>0.009844</td>\n",
       "      <td>0.033587</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95409</th>\n",
       "      <td>1368.550962</td>\n",
       "      <td>-367.424548</td>\n",
       "      <td>-195.877313</td>\n",
       "      <td>-325.880815</td>\n",
       "      <td>-388.317946</td>\n",
       "      <td>194.174788</td>\n",
       "      <td>-14.036008</td>\n",
       "      <td>-774.801858</td>\n",
       "      <td>-290.839044</td>\n",
       "      <td>2278.121890</td>\n",
       "      <td>...</td>\n",
       "      <td>0.095174</td>\n",
       "      <td>0.197797</td>\n",
       "      <td>0.122209</td>\n",
       "      <td>-0.000735</td>\n",
       "      <td>0.120857</td>\n",
       "      <td>-0.158416</td>\n",
       "      <td>-0.051092</td>\n",
       "      <td>0.001329</td>\n",
       "      <td>-0.231483</td>\n",
       "      <td>0.176777</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95410</th>\n",
       "      <td>2770.205041</td>\n",
       "      <td>-537.118495</td>\n",
       "      <td>-92.373827</td>\n",
       "      <td>352.557972</td>\n",
       "      <td>-640.184514</td>\n",
       "      <td>-15.855259</td>\n",
       "      <td>191.368300</td>\n",
       "      <td>-1379.473664</td>\n",
       "      <td>-1192.886720</td>\n",
       "      <td>3617.889383</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.063948</td>\n",
       "      <td>0.076816</td>\n",
       "      <td>0.026445</td>\n",
       "      <td>-0.034682</td>\n",
       "      <td>-0.050798</td>\n",
       "      <td>0.052023</td>\n",
       "      <td>-0.034453</td>\n",
       "      <td>0.098954</td>\n",
       "      <td>-0.009225</td>\n",
       "      <td>-0.044114</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95411</th>\n",
       "      <td>2407.241426</td>\n",
       "      <td>-449.248569</td>\n",
       "      <td>114.894250</td>\n",
       "      <td>34.853700</td>\n",
       "      <td>-416.627509</td>\n",
       "      <td>163.516005</td>\n",
       "      <td>-176.851370</td>\n",
       "      <td>-1571.324151</td>\n",
       "      <td>-1236.132672</td>\n",
       "      <td>3716.891888</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.215516</td>\n",
       "      <td>-0.104608</td>\n",
       "      <td>0.003286</td>\n",
       "      <td>-0.224950</td>\n",
       "      <td>0.055244</td>\n",
       "      <td>0.007030</td>\n",
       "      <td>-0.042468</td>\n",
       "      <td>-0.009838</td>\n",
       "      <td>0.018166</td>\n",
       "      <td>-0.009733</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>95412 rows × 607 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               0            1           2            3            4    \\\n",
       "0       831.883227  -646.861895 -363.955316  -182.229876  -400.367679   \n",
       "1      3574.424159 -1134.829920 -210.071125  1506.846801  -710.712659   \n",
       "2       681.973233  -547.511041  -62.104065  -152.397865  -532.179443   \n",
       "3      2248.176026  -371.333556   17.341024   153.031358  -559.494031   \n",
       "4      1424.241454  -326.772925 -135.899676  -236.083293  -504.559563   \n",
       "...            ...          ...         ...          ...          ...   \n",
       "95407  1466.972344  -537.481623  417.465311  -241.452684 -1178.093742   \n",
       "95408  2345.345493  -803.998002   73.513108   406.864479  -728.035887   \n",
       "95409  1368.550962  -367.424548 -195.877313  -325.880815  -388.317946   \n",
       "95410  2770.205041  -537.118495  -92.373827   352.557972  -640.184514   \n",
       "95411  2407.241426  -449.248569  114.894250    34.853700  -416.627509   \n",
       "\n",
       "               5           6            7            8            9    ...  \\\n",
       "0      -381.638509  265.626351   277.234640    21.686036  3375.632218  ...   \n",
       "1     -1000.928500  697.962871  -156.237400  -541.669607  2921.697086  ...   \n",
       "2      -663.551389  -36.567874   297.875608  -238.958811  3497.992065  ...   \n",
       "3       202.556490   61.571859 -1623.052607 -1222.283285  3686.672549  ...   \n",
       "4         9.072990  -37.746297  -853.493295  -824.684647  3658.558861  ...   \n",
       "...            ...         ...          ...          ...          ...  ...   \n",
       "95407 -1058.511164 -443.431369   560.561072  -127.990495 -1053.981987  ...   \n",
       "95408   -55.330810  184.177274  -172.615589   535.689080 -2955.636037  ...   \n",
       "95409   194.174788  -14.036008  -774.801858  -290.839044  2278.121890  ...   \n",
       "95410   -15.855259  191.368300 -1379.473664 -1192.886720  3617.889383  ...   \n",
       "95411   163.516005 -176.851370 -1571.324151 -1236.132672  3716.891888  ...   \n",
       "\n",
       "            530       531       532       533       534       535       536  \\\n",
       "0      0.055177  0.058121  0.016822 -0.056233 -0.055087  0.007557 -0.008825   \n",
       "1      0.020850  0.008793  0.002238 -0.010681  0.049743  0.065684 -0.025363   \n",
       "2     -0.135746  0.069506 -0.141231 -0.040676  0.057974  0.053992  0.123118   \n",
       "3     -0.018941 -0.025626 -0.028587 -0.018650  0.068444 -0.033556  0.028733   \n",
       "4     -0.047736 -0.013447  0.021439  0.078648  0.003598  0.050231 -0.010456   \n",
       "...         ...       ...       ...       ...       ...       ...       ...   \n",
       "95407 -0.016459 -0.006181  0.025597  0.002865 -0.031965  0.010783 -0.007026   \n",
       "95408  0.001280  0.047298 -0.063582 -0.011937  0.017651  0.010413 -0.018632   \n",
       "95409  0.095174  0.197797  0.122209 -0.000735  0.120857 -0.158416 -0.051092   \n",
       "95410 -0.063948  0.076816  0.026445 -0.034682 -0.050798  0.052023 -0.034453   \n",
       "95411 -0.215516 -0.104608  0.003286 -0.224950  0.055244  0.007030 -0.042468   \n",
       "\n",
       "            537       538       539  \n",
       "0     -0.036627  0.016017  0.082159  \n",
       "1     -0.030930 -0.004446 -0.015777  \n",
       "2      0.051522 -0.030138  0.116640  \n",
       "3      0.031233 -0.019987 -0.081835  \n",
       "4      0.005339  0.085180  0.138701  \n",
       "...         ...       ...       ...  \n",
       "95407 -0.029539  0.007391 -0.014890  \n",
       "95408  0.048859  0.009844  0.033587  \n",
       "95409  0.001329 -0.231483  0.176777  \n",
       "95410  0.098954 -0.009225 -0.044114  \n",
       "95411 -0.009838  0.018166 -0.009733  \n",
       "\n",
       "[95412 rows x 607 columns]"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Here I concat the dataframes\n",
    "x = pd.concat([df, f], axis=1)\n",
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "ae804fb9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0      float64\n",
       "1      float64\n",
       "2      float64\n",
       "3      float64\n",
       "4      float64\n",
       "        ...   \n",
       "535    float64\n",
       "536    float64\n",
       "537    float64\n",
       "538    float64\n",
       "539    float64\n",
       "Length: 607, dtype: object"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "35ce03b2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>530</th>\n",
       "      <th>531</th>\n",
       "      <th>532</th>\n",
       "      <th>533</th>\n",
       "      <th>534</th>\n",
       "      <th>535</th>\n",
       "      <th>536</th>\n",
       "      <th>537</th>\n",
       "      <th>538</th>\n",
       "      <th>539</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>831.883227</td>\n",
       "      <td>-646.861895</td>\n",
       "      <td>-363.955316</td>\n",
       "      <td>-182.229876</td>\n",
       "      <td>-400.367679</td>\n",
       "      <td>-381.638509</td>\n",
       "      <td>265.626351</td>\n",
       "      <td>277.234640</td>\n",
       "      <td>21.686036</td>\n",
       "      <td>3375.632218</td>\n",
       "      <td>...</td>\n",
       "      <td>0.055177</td>\n",
       "      <td>0.058121</td>\n",
       "      <td>0.016822</td>\n",
       "      <td>-0.056233</td>\n",
       "      <td>-0.055087</td>\n",
       "      <td>0.007557</td>\n",
       "      <td>-0.008825</td>\n",
       "      <td>-0.036627</td>\n",
       "      <td>0.016017</td>\n",
       "      <td>0.082159</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3574.424159</td>\n",
       "      <td>-1134.829920</td>\n",
       "      <td>-210.071125</td>\n",
       "      <td>1506.846801</td>\n",
       "      <td>-710.712659</td>\n",
       "      <td>-1000.928500</td>\n",
       "      <td>697.962871</td>\n",
       "      <td>-156.237400</td>\n",
       "      <td>-541.669607</td>\n",
       "      <td>2921.697086</td>\n",
       "      <td>...</td>\n",
       "      <td>0.020850</td>\n",
       "      <td>0.008793</td>\n",
       "      <td>0.002238</td>\n",
       "      <td>-0.010681</td>\n",
       "      <td>0.049743</td>\n",
       "      <td>0.065684</td>\n",
       "      <td>-0.025363</td>\n",
       "      <td>-0.030930</td>\n",
       "      <td>-0.004446</td>\n",
       "      <td>-0.015777</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>681.973233</td>\n",
       "      <td>-547.511041</td>\n",
       "      <td>-62.104065</td>\n",
       "      <td>-152.397865</td>\n",
       "      <td>-532.179443</td>\n",
       "      <td>-663.551389</td>\n",
       "      <td>-36.567874</td>\n",
       "      <td>297.875608</td>\n",
       "      <td>-238.958811</td>\n",
       "      <td>3497.992065</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.135746</td>\n",
       "      <td>0.069506</td>\n",
       "      <td>-0.141231</td>\n",
       "      <td>-0.040676</td>\n",
       "      <td>0.057974</td>\n",
       "      <td>0.053992</td>\n",
       "      <td>0.123118</td>\n",
       "      <td>0.051522</td>\n",
       "      <td>-0.030138</td>\n",
       "      <td>0.116640</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2248.176026</td>\n",
       "      <td>-371.333556</td>\n",
       "      <td>17.341024</td>\n",
       "      <td>153.031358</td>\n",
       "      <td>-559.494031</td>\n",
       "      <td>202.556490</td>\n",
       "      <td>61.571859</td>\n",
       "      <td>-1623.052607</td>\n",
       "      <td>-1222.283285</td>\n",
       "      <td>3686.672549</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.018941</td>\n",
       "      <td>-0.025626</td>\n",
       "      <td>-0.028587</td>\n",
       "      <td>-0.018650</td>\n",
       "      <td>0.068444</td>\n",
       "      <td>-0.033556</td>\n",
       "      <td>0.028733</td>\n",
       "      <td>0.031233</td>\n",
       "      <td>-0.019987</td>\n",
       "      <td>-0.081835</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1424.241454</td>\n",
       "      <td>-326.772925</td>\n",
       "      <td>-135.899676</td>\n",
       "      <td>-236.083293</td>\n",
       "      <td>-504.559563</td>\n",
       "      <td>9.072990</td>\n",
       "      <td>-37.746297</td>\n",
       "      <td>-853.493295</td>\n",
       "      <td>-824.684647</td>\n",
       "      <td>3658.558861</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.047736</td>\n",
       "      <td>-0.013447</td>\n",
       "      <td>0.021439</td>\n",
       "      <td>0.078648</td>\n",
       "      <td>0.003598</td>\n",
       "      <td>0.050231</td>\n",
       "      <td>-0.010456</td>\n",
       "      <td>0.005339</td>\n",
       "      <td>0.085180</td>\n",
       "      <td>0.138701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95407</th>\n",
       "      <td>1466.972344</td>\n",
       "      <td>-537.481623</td>\n",
       "      <td>417.465311</td>\n",
       "      <td>-241.452684</td>\n",
       "      <td>-1178.093742</td>\n",
       "      <td>-1058.511164</td>\n",
       "      <td>-443.431369</td>\n",
       "      <td>560.561072</td>\n",
       "      <td>-127.990495</td>\n",
       "      <td>-1053.981987</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.016459</td>\n",
       "      <td>-0.006181</td>\n",
       "      <td>0.025597</td>\n",
       "      <td>0.002865</td>\n",
       "      <td>-0.031965</td>\n",
       "      <td>0.010783</td>\n",
       "      <td>-0.007026</td>\n",
       "      <td>-0.029539</td>\n",
       "      <td>0.007391</td>\n",
       "      <td>-0.014890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95408</th>\n",
       "      <td>2345.345493</td>\n",
       "      <td>-803.998002</td>\n",
       "      <td>73.513108</td>\n",
       "      <td>406.864479</td>\n",
       "      <td>-728.035887</td>\n",
       "      <td>-55.330810</td>\n",
       "      <td>184.177274</td>\n",
       "      <td>-172.615589</td>\n",
       "      <td>535.689080</td>\n",
       "      <td>-2955.636037</td>\n",
       "      <td>...</td>\n",
       "      <td>0.001280</td>\n",
       "      <td>0.047298</td>\n",
       "      <td>-0.063582</td>\n",
       "      <td>-0.011937</td>\n",
       "      <td>0.017651</td>\n",
       "      <td>0.010413</td>\n",
       "      <td>-0.018632</td>\n",
       "      <td>0.048859</td>\n",
       "      <td>0.009844</td>\n",
       "      <td>0.033587</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95409</th>\n",
       "      <td>1368.550962</td>\n",
       "      <td>-367.424548</td>\n",
       "      <td>-195.877313</td>\n",
       "      <td>-325.880815</td>\n",
       "      <td>-388.317946</td>\n",
       "      <td>194.174788</td>\n",
       "      <td>-14.036008</td>\n",
       "      <td>-774.801858</td>\n",
       "      <td>-290.839044</td>\n",
       "      <td>2278.121890</td>\n",
       "      <td>...</td>\n",
       "      <td>0.095174</td>\n",
       "      <td>0.197797</td>\n",
       "      <td>0.122209</td>\n",
       "      <td>-0.000735</td>\n",
       "      <td>0.120857</td>\n",
       "      <td>-0.158416</td>\n",
       "      <td>-0.051092</td>\n",
       "      <td>0.001329</td>\n",
       "      <td>-0.231483</td>\n",
       "      <td>0.176777</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95410</th>\n",
       "      <td>2770.205041</td>\n",
       "      <td>-537.118495</td>\n",
       "      <td>-92.373827</td>\n",
       "      <td>352.557972</td>\n",
       "      <td>-640.184514</td>\n",
       "      <td>-15.855259</td>\n",
       "      <td>191.368300</td>\n",
       "      <td>-1379.473664</td>\n",
       "      <td>-1192.886720</td>\n",
       "      <td>3617.889383</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.063948</td>\n",
       "      <td>0.076816</td>\n",
       "      <td>0.026445</td>\n",
       "      <td>-0.034682</td>\n",
       "      <td>-0.050798</td>\n",
       "      <td>0.052023</td>\n",
       "      <td>-0.034453</td>\n",
       "      <td>0.098954</td>\n",
       "      <td>-0.009225</td>\n",
       "      <td>-0.044114</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95411</th>\n",
       "      <td>2407.241426</td>\n",
       "      <td>-449.248569</td>\n",
       "      <td>114.894250</td>\n",
       "      <td>34.853700</td>\n",
       "      <td>-416.627509</td>\n",
       "      <td>163.516005</td>\n",
       "      <td>-176.851370</td>\n",
       "      <td>-1571.324151</td>\n",
       "      <td>-1236.132672</td>\n",
       "      <td>3716.891888</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.215516</td>\n",
       "      <td>-0.104608</td>\n",
       "      <td>0.003286</td>\n",
       "      <td>-0.224950</td>\n",
       "      <td>0.055244</td>\n",
       "      <td>0.007030</td>\n",
       "      <td>-0.042468</td>\n",
       "      <td>-0.009838</td>\n",
       "      <td>0.018166</td>\n",
       "      <td>-0.009733</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>95412 rows × 607 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               0            1           2            3            4    \\\n",
       "0       831.883227  -646.861895 -363.955316  -182.229876  -400.367679   \n",
       "1      3574.424159 -1134.829920 -210.071125  1506.846801  -710.712659   \n",
       "2       681.973233  -547.511041  -62.104065  -152.397865  -532.179443   \n",
       "3      2248.176026  -371.333556   17.341024   153.031358  -559.494031   \n",
       "4      1424.241454  -326.772925 -135.899676  -236.083293  -504.559563   \n",
       "...            ...          ...         ...          ...          ...   \n",
       "95407  1466.972344  -537.481623  417.465311  -241.452684 -1178.093742   \n",
       "95408  2345.345493  -803.998002   73.513108   406.864479  -728.035887   \n",
       "95409  1368.550962  -367.424548 -195.877313  -325.880815  -388.317946   \n",
       "95410  2770.205041  -537.118495  -92.373827   352.557972  -640.184514   \n",
       "95411  2407.241426  -449.248569  114.894250    34.853700  -416.627509   \n",
       "\n",
       "               5           6            7            8            9    ...  \\\n",
       "0      -381.638509  265.626351   277.234640    21.686036  3375.632218  ...   \n",
       "1     -1000.928500  697.962871  -156.237400  -541.669607  2921.697086  ...   \n",
       "2      -663.551389  -36.567874   297.875608  -238.958811  3497.992065  ...   \n",
       "3       202.556490   61.571859 -1623.052607 -1222.283285  3686.672549  ...   \n",
       "4         9.072990  -37.746297  -853.493295  -824.684647  3658.558861  ...   \n",
       "...            ...         ...          ...          ...          ...  ...   \n",
       "95407 -1058.511164 -443.431369   560.561072  -127.990495 -1053.981987  ...   \n",
       "95408   -55.330810  184.177274  -172.615589   535.689080 -2955.636037  ...   \n",
       "95409   194.174788  -14.036008  -774.801858  -290.839044  2278.121890  ...   \n",
       "95410   -15.855259  191.368300 -1379.473664 -1192.886720  3617.889383  ...   \n",
       "95411   163.516005 -176.851370 -1571.324151 -1236.132672  3716.891888  ...   \n",
       "\n",
       "            530       531       532       533       534       535       536  \\\n",
       "0      0.055177  0.058121  0.016822 -0.056233 -0.055087  0.007557 -0.008825   \n",
       "1      0.020850  0.008793  0.002238 -0.010681  0.049743  0.065684 -0.025363   \n",
       "2     -0.135746  0.069506 -0.141231 -0.040676  0.057974  0.053992  0.123118   \n",
       "3     -0.018941 -0.025626 -0.028587 -0.018650  0.068444 -0.033556  0.028733   \n",
       "4     -0.047736 -0.013447  0.021439  0.078648  0.003598  0.050231 -0.010456   \n",
       "...         ...       ...       ...       ...       ...       ...       ...   \n",
       "95407 -0.016459 -0.006181  0.025597  0.002865 -0.031965  0.010783 -0.007026   \n",
       "95408  0.001280  0.047298 -0.063582 -0.011937  0.017651  0.010413 -0.018632   \n",
       "95409  0.095174  0.197797  0.122209 -0.000735  0.120857 -0.158416 -0.051092   \n",
       "95410 -0.063948  0.076816  0.026445 -0.034682 -0.050798  0.052023 -0.034453   \n",
       "95411 -0.215516 -0.104608  0.003286 -0.224950  0.055244  0.007030 -0.042468   \n",
       "\n",
       "            537       538       539  \n",
       "0     -0.036627  0.016017  0.082159  \n",
       "1     -0.030930 -0.004446 -0.015777  \n",
       "2      0.051522 -0.030138  0.116640  \n",
       "3      0.031233 -0.019987 -0.081835  \n",
       "4      0.005339  0.085180  0.138701  \n",
       "...         ...       ...       ...  \n",
       "95407 -0.029539  0.007391 -0.014890  \n",
       "95408  0.048859  0.009844  0.033587  \n",
       "95409  0.001329 -0.231483  0.176777  \n",
       "95410  0.098954 -0.009225 -0.044114  \n",
       "95411 -0.009838  0.018166 -0.009733  \n",
       "\n",
       "[95412 rows x 607 columns]"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.astype(float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "f30d5829",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(95412, 607)"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Because I data['TARGET_D'] is not integreted in the dataframe \n",
    "#I concate it with the rest of the data\n",
    "\n",
    "x = x.astype(float)\n",
    "x.dtypes\n",
    "set(x.dtypes)\n",
    "x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "fd9106a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "y = data['TARGET_B']\n",
    "y = data['TARGET_B'].astype(float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "b61a1754",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dtype('float64')"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.dtypes"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ca7a2cff",
   "metadata": {},
   "source": [
    "Making the models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "61a13ffb",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Starting with the benchmark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "c1de5408",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(max_iter=10000, random_state=42)"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2, random_state=42)\n",
    "\n",
    "classification = LogisticRegression(random_state=42, max_iter=10000)\n",
    "classification.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "917edd64",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9487501965099827"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "classification.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "98c14e76",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import classification_report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "085a7991",
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = classification.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "1d1a0632",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([  0. ,  10. ,  15. ,   9. ,   7. ,  20. ,  25. , 100. ,  21. ,\n",
       "         8. ,   5. ,  30. ,  17. ,  14. ,  13. ,   6. ,  46. ,   2. ,\n",
       "        50. ,  16. ,  11. ,   4. ,  23. ,   3. ,  18. ,  26. ,  27. ,\n",
       "         1. ,  12. ,  41. ,  35. ,  32. ,  22. ,  19. ,  40. ,  12.5,\n",
       "         7.5, 200. ,  36. ,  28. ,  24. ,  38. ,  45. , 102. ])"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.loc[y_test.index]['TARGET_D'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "4dfcb4e8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "This model is not worth it at the moment. Just mail everyone! Savings: -2466.56\n"
     ]
    }
   ],
   "source": [
    "df = pd.DataFrame()\n",
    "df['send_mail'] = predictions\n",
    "df['TARGET_D'] = file.loc[y_test.index]['TARGET_D'].to_list()\n",
    "df['TARGET_B'] = file.loc[y_test.index]['TARGET_B'].to_list()\n",
    "\n",
    "total_amount_spent_on_mail = len(df[df['send_mail'] > 0]) * 0.68\n",
    "total_amount_spent_on_mail_everyone = len(df) * 0.68\n",
    "total_missed_donations = sum(df[(df['send_mail'] == 0) & (df['TARGET_D'] > 0)]['TARGET_D'])\n",
    "total_extra_spent_on_mail = round(len(df[(df['send_mail'] == 1) & (df['TARGET_B'] == 0)]) * 0.68, 2)\n",
    "\n",
    "savings = round(total_amount_spent_on_mail_everyone - total_missed_donations - total_amount_spent_on_mail, 2)\n",
    "\n",
    "if savings <= 0:\n",
    "    print(f'This model is not worth it at the moment. Just mail everyone! Savings: {savings}')\n",
    "else:\n",
    "    print(f'''This model saved ${savings} in mails.\n",
    "        Missed ${total_missed_donations} in donations and wasted ${total_extra_spent_on_mail} in mails not responded.\n",
    "        You made: ${sum(df['TARGET_D']) - total_amount_spent_on_mail}\n",
    "           ''')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "266d5d1e",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Before I will make model 2 I will scale the data first\n",
    "#Upsampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "0f7ea120",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(class_weight={1: 3}, max_iter=10000, random_state=42)"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Using the scaled data and do something with the class_weight={1:3}\n",
    "X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2, random_state=42)\n",
    "\n",
    "from imblearn.over_sampling import SMOTE\n",
    "\n",
    "smote = SMOTE()\n",
    "\n",
    "X_sm, y_sm = smote.fit_resample (np.array(X_train), np.array(y_train))\n",
    "\n",
    "classification = LogisticRegression(random_state=42, class_weight={1:3}, max_iter=10000)\n",
    "classification.fit(X_sm, y_sm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "b3354bf1",
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = classification.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "164283fa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "This model is not worth it at the moment. Just mail everyone! Savings: -175.68\n"
     ]
    }
   ],
   "source": [
    "# This is a better model\n",
    "df = pd.DataFrame()\n",
    "df['send_mail'] = predictions\n",
    "df['TARGET_D'] = file.loc[y_test.index]['TARGET_D'].to_list()\n",
    "df['TARGET_B'] = file.loc[y_test.index]['TARGET_B'].to_list()\n",
    "\n",
    "total_amount_spent_on_mail = len(df[df['send_mail'] > 0]) * 0.68\n",
    "total_amount_spent_on_mail_everyone = len(df) * 0.68\n",
    "total_missed_donations = sum(df[(df['send_mail'] == 0) & (df['TARGET_D'] > 0)]['TARGET_D'])\n",
    "total_extra_spent_on_mail = round(len(df[(df['send_mail'] == 1) & (df['TARGET_B'] == 0)]) * 0.68, 2)\n",
    "\n",
    "savings = round(total_amount_spent_on_mail_everyone - total_missed_donations - total_amount_spent_on_mail, 2)\n",
    "\n",
    "if savings <= 0:\n",
    "    print(f'This model is not worth it at the moment. Just mail everyone! Savings: {savings}')\n",
    "else:\n",
    "    print(f'''This model saved ${savings} in mails.\n",
    "        Missed ${total_missed_donations} in donations and wasted ${total_extra_spent_on_mail} in mails not responded.\n",
    "        You made: ${sum(df['TARGET_D']) - total_amount_spent_on_mail}\n",
    "           ''')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "400a9f5e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "         0.0       0.95      1.00      0.97     27200\n",
      "         1.0       0.10      0.00      0.00      1424\n",
      "\n",
      "    accuracy                           0.95     28624\n",
      "   macro avg       0.53      0.50      0.49     28624\n",
      "weighted avg       0.91      0.95      0.93     28624\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#Trying the RandomForestClassifier and use the data that is sampled\n",
    "from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor\n",
    "from sklearn.metrics import classification_report\n",
    "\n",
    "model = RandomForestClassifier()\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.3, random_state=42)\n",
    "\n",
    "smote = SMOTE()\n",
    "\n",
    "X_sm, y_sm = smote.fit_resample (np.array(X_train), np.array(y_train))\n",
    "\n",
    "model.fit(X_sm,y_sm)\n",
    "\n",
    "predictions = model.predict(X_test)\n",
    "\n",
    "print(classification_report(y_test, predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "0edc17e0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "This model is not worth it at the moment. Just mail everyone! Savings: -2966.33\n"
     ]
    }
   ],
   "source": [
    "# This model is not worth using \n",
    "df = pd.DataFrame()\n",
    "df['send_mail'] = predictions\n",
    "df['TARGET_D'] = file.loc[y_test.index]['TARGET_D'].to_list()\n",
    "df['TARGET_B'] = file.loc[y_test.index]['TARGET_B'].to_list()\n",
    "\n",
    "total_amount_spent_on_mail = len(df[df['send_mail'] > 0]) * 0.68\n",
    "total_amount_spent_on_mail_everyone = len(df) * 0.68\n",
    "total_missed_donations = sum(df[(df['send_mail'] == 0) & (df['TARGET_D'] > 0)]['TARGET_D'])\n",
    "total_extra_spent_on_mail = round(len(df[(df['send_mail'] == 1) & (df['TARGET_B'] == 0)]) * 0.68, 2)\n",
    "\n",
    "savings = round(total_amount_spent_on_mail_everyone - total_missed_donations - total_amount_spent_on_mail, 2)\n",
    "\n",
    "if savings <= 0:\n",
    "    print(f'This model is not worth it at the moment. Just mail everyone! Savings: {savings}')\n",
    "else:\n",
    "    print(f'''This model saved ${savings} in mails.\n",
    "        Missed ${total_missed_donations} in donations and wasted ${total_extra_spent_on_mail} in mails not responded.\n",
    "        You made: ${sum(df['TARGET_D']) - total_amount_spent_on_mail}\n",
    "           ''')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "31a928db",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.25,\n",
    "                                                    stratify=y, random_state=0)\n",
    "\n",
    "smote = SMOTE()\n",
    "\n",
    "X_sm, y_sm = smote.fit_resample (np.array(X_train), np.array(y_train))\n",
    "\n",
    "knn = KNeighborsClassifier()\n",
    "knn.fit(X_sm,y_sm)\n",
    "predictions = knn.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "aff3f272",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "This model is not worth it at the moment. Just mail everyone! Savings: -1684.06\n"
     ]
    }
   ],
   "source": [
    "df = pd.DataFrame()\n",
    "df['send_mail'] = predictions\n",
    "df['TARGET_D'] = file.loc[y_test.index]['TARGET_D'].to_list()\n",
    "df['TARGET_B'] = file.loc[y_test.index]['TARGET_B'].to_list()\n",
    "\n",
    "total_amount_spent_on_mail = len(df[df['send_mail'] > 0]) * 0.68\n",
    "total_amount_spent_on_mail_everyone = len(df) * 0.68\n",
    "total_missed_donations = sum(df[(df['send_mail'] == 0) & (df['TARGET_D'] > 0)]['TARGET_D'])\n",
    "total_extra_spent_on_mail = round(len(df[(df['send_mail'] == 1) & (df['TARGET_B'] == 0)]) * 0.68, 2)\n",
    "\n",
    "savings = round(total_amount_spent_on_mail_everyone - total_missed_donations - total_amount_spent_on_mail, 2)\n",
    "\n",
    "if savings <= 0:\n",
    "    print(f'This model is not worth it at the moment. Just mail everyone! Savings: {savings}')\n",
    "else:\n",
    "    print(f'''This model saved ${savings} in mails.\n",
    "        Missed ${total_missed_donations} in donations and wasted ${total_extra_spent_on_mail} in mails not responded.\n",
    "        You made: ${sum(df['TARGET_D']) - total_amount_spent_on_mail}\n",
    "           ''')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "32be1735",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Conclussion: No one of my models I use. I could best mail everyone "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}