Preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Importing Necessary Modules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "import sys\n",
    "import scipy as sc\n",
    "pd.set_option('display.max_columns', None)\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def file_reader(data):\n",
    "    return pd.read_csv(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = file_reader('Train.csv')\n",
    "test = file_reader('Test.csv')\n",
    "variable = file_reader('VariableDefinitions (1).csv').T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Creating a Sample of the Whole DataSet for EDA and Feature Engineering\n",
    "\n",
    "#I had to do this to reduce computation \n",
    "\n",
    "#Since i was using a cpu and not Google Colab or Other Kernels that give out free GPU Usage\n",
    "\n",
    "#df =train.sample(frac= 0.15).reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#This Column should be run after you have finish exploring the dataset and creating feature for ease of computation\n",
    "\n",
    "#when running this column do not run the column of the sample above hence comment the column to prevent change in variable name\n",
    "\n",
    "df = pd.concat([train,test],ignore_index=True, sort=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#The Varibale name of each of the column to\n",
    "\n",
    "variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def churn_percentage(df, data):\n",
    "    '''\n",
    "    To Check if the newly created train sample captures \n",
    "    both the Churned and Not Churned in the respected ratio\n",
    "    '''\n",
    "    data = data['CHURN'].value_counts()\n",
    "    data_0 = data[0]\n",
    "    data_1 = data[1]\n",
    "    dataset = 100 * (data_1 /(data_0 + data_1))\n",
    "    \n",
    "    df= df['CHURN'].value_counts()\n",
    "    df_0 = df[0]\n",
    "    df_1 = df[1]\n",
    "    sample_df = 100 * (df_1 /(df_0 + df_1))\n",
    "    \n",
    "    print(f'Percentage of Churned of Sample data is {sample_df}\\n\\nWhile Percentage of Churned of Whole Datasetet is {dataset} ')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "churn_percentage(df, train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Exploratory Data Analysis (EDA)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Checking for duplicated columns and rows\n",
    "df.duplicated().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Shape of the data\n",
    "def shape(data):\n",
    "    print(f'This is the shape of the df dataset: {data.shape}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "shape(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Checking out the Categorical Column and Numerical in the Dataset\n",
    "def cat_num(data):\n",
    "    cat_col = []\n",
    "    num_col = []\n",
    "    for feat in data.columns:\n",
    "        if data[feat].dtypes =='O':\n",
    "            cat_col.append(feat)\n",
    "        else:\n",
    "            num_col.append(feat)\n",
    "    return cat_col, num_col"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#This columns where dropped because there are all indentical\n",
    "df = df.drop(labels = ['user_id','MRG'], axis= 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def min_max(column):\n",
    "    print(f'Max {df[column].max()}\\n\\nMin {df[column].min()}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "min_max(\"REVENUE\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.TENURE.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def time_frame(data):\n",
    "    new_var = []\n",
    "    new_list = [i for i in data['TOP_PACK']]\n",
    "    for val in new_list:\n",
    "        val = str(val)\n",
    "        if '_' and ';' in val:\n",
    "            new_var.append(val.split(';')[-1])\n",
    "        elif '_' in val:\n",
    "            new_var.append(val.split('_')[-1])\n",
    "        elif ';' in val:\n",
    "            new_var.append(val.split(';')[-1])\n",
    "        elif ',' in val:\n",
    "            new_var.append(val.split(',')[-1])\n",
    "        elif 'nan' in val:\n",
    "            new_var.append(np.nan)\n",
    "        else:\n",
    "            new_var.append(val)\n",
    "    return new_var"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['TIME_FRAME'] = time_frame(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['TIME_FRAME'] = df['TIME_FRAME'].str.lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def data_call(data):\n",
    "    data_type = [i for i in data['TIME_FRAME']]\n",
    "    data_cal = []\n",
    "    for val in data_type:\n",
    "        val = str(val)\n",
    "        if 'call' in val:\n",
    "            data_cal.append(1)\n",
    "        elif 'nan' in val:\n",
    "            data_cal.append(np.nan)\n",
    "        else:\n",
    "            data_cal.append(0)\n",
    "    return data_cal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def unlimited(data):\n",
    "    limi =[]\n",
    "    __ = [i for i in data['TOP_PACK']]\n",
    "    for value in __:\n",
    "        value = str(value)\n",
    "        if 'limi' in value:\n",
    "            limi.append(1)\n",
    "        elif 'nan' in value:\n",
    "            limi.append(np.nan)\n",
    "        else:\n",
    "            limi.append(0)\n",
    "    return limi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['DATA_CAPPED'] unlimited(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def data_type_fix(data):\n",
    "    data_type = [i for i in df['TOP_PACK'].str.split().str[0].str.lower()]\n",
    "    fix_data = []\n",
    "    for value in data_type:\n",
    "        value = str(value)\n",
    "        if 'data' in value:\n",
    "            fix_data.append('data')\n",
    "        elif 'on' in value:\n",
    "            fix_data.append('on-net')\n",
    "        elif 'wifi' in value:\n",
    "            fix_data.append('wifi')\n",
    "        elif 'all-net' in value:\n",
    "            fix_data.append('all-net')\n",
    "        elif 'nan' in value:\n",
    "            fix_data.append(np.nan)\n",
    "        elif 'ivr' in value:\n",
    "            fix_data.append('var_ivr')\n",
    "        elif 'twter' in value:\n",
    "            fix_data.append('twter')\n",
    "        elif 'jokk' in value:\n",
    "            fix_data.append('jokko')\n",
    "        elif 'mixt' in value:\n",
    "            fix_data.append('mixt')\n",
    "        else:\n",
    "            fix_data.append('others')\n",
    "    return fix_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['DATA_CALL'] = data_call(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['DATA_TYPE'] = data_type_fix(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def time_range(data):\n",
    "    data_type = [i for i in data['TIME_FRAME']]\n",
    "    fix_data = []\n",
    "    for val in data_type:\n",
    "        val = str(val)\n",
    "        if 'd' in val[-1] and 'i' not in val:\n",
    "            fix_data.append(int(val[:-1]))\n",
    "        elif 'mo' in val:\n",
    "            fix_data.append(30)\n",
    "        elif 'wee' in val:\n",
    "            fix_data.append(7)\n",
    "        elif '48h' == val:\n",
    "            fix_data.append(2)\n",
    "        elif 'nan' in val:\n",
    "            fix_data.append(np.nan)\n",
    "        elif '4h' in val or 'da' in val or '0h' in val:\n",
    "            fix_data.append(1)\n",
    "        else:\n",
    "            fix_data.append(4)\n",
    "    return fix_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['TIME_RANGE'] = time_range(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Time Rate\n",
    "\n",
    "def time_rate(data):\n",
    "    rate =[]\n",
    "    __ = [i for i in data['TIME_FRAME']]\n",
    "    for val in __:\n",
    "        val = str(val)\n",
    "        if 'h' in val:\n",
    "            rate.append(1)\n",
    "        elif 'dai' in val:\n",
    "            rate.append(2)\n",
    "        elif 'wee' in val:\n",
    "            rate.append(3)\n",
    "        elif 'mon' in val:\n",
    "            rate.append(5)\n",
    "        elif 'nan' in val:\n",
    "            rate.append(np.nan)\n",
    "        elif 'd' in val and 'i' not in val:\n",
    "            rate.append(4)\n",
    "        else:\n",
    "            rate.append(0)\n",
    "    return rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['DATA_RATE'] = time_rate(df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Converting the Categorical column to Numeric Values using Label Encoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import LabelEncoder\n",
    "le = LabelEncoder()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.REGION = df.REGION.fillna('unknown')\n",
    "df.DATA_TYPE = df.DATA_TYPE.fillna('unknown')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['REGION'] = le.fit_transform(df.REGION).astype(np.uint8)\n",
    "df.TENURE = le.fit_transform(df.TENURE).astype(np.uint8)\n",
    "df.DATA_TYPE = le.fit_transform(df.DATA_TYPE).astype(np.uint8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('new_dataset.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Possible Improvement The Region Column would be better to use Pandas Get Dummy Mehthod</h3>\n",
    "\n",
    "<b>I notice this late towards the end of the competition and decided to stick with the Label Encoder</b>"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}