preprocess_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import pickle\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def transform(ds: str, suffix:str='', sep:str=';', ext:str='csv', drop:list=None, cat_names:list=None, to_disk:bool=True, target:str='', d_basepath='data'):\n",
    "    if target == '':  target = False\n",
    "    if drop is None: drop = []\n",
    "    if cat_names is None: cat_names = []\n",
    "        \n",
    "    base_fname = f'{d_basepath}/{ds}/{ds}'\n",
    "    source_fname = f'{base_fname}{suffix}.{ext}'\n",
    "    print(f'Basepath: {base_fname}')\n",
    "    print(f'Source file: {source_fname}')\n",
    "    fname = os.path.basename(os.path.splitext(source_fname)[0])\n",
    "    \n",
    "    df = pd.read_csv(source_fname, sep=sep)\n",
    "    df = df.drop(drop, axis=1)\n",
    "    df_num, subs = cat_to_num(df, cat_names=cat_names)\n",
    "    pickle.dump(subs, open(f'{d_basepath}/{ds}/subs.pkl', 'wb'))\n",
    "\n",
    "    if target:\n",
    "        y = df_num[target]\n",
    "        df_num = df_num.drop([target], axis=1)\n",
    "        \n",
    "    if to_disk:\n",
    "        if target:\n",
    "            target_fname_y = f'{base_fname}_labels.csv'\n",
    "            print(f'Target file label: {target_fname_y}')\n",
    "            y.to_csv(target_fname_y, sep=';', index=False)\n",
    "        target_fname = f'{base_fname}.csv'\n",
    "        print(f'Target file: {target_fname}')\n",
    "        df_num.to_csv(target_fname, sep=';', index=False)\n",
    "    if target:\n",
    "        return df_num, y, subs\n",
    "    return df_num, subs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cat_to_num(df, sep=',', cat_names=None):\n",
    "    if cat_names is None: cat_names = []\n",
    "    subs = {}\n",
    "    df_num = df.copy()\n",
    "    \n",
    "    # TRANSFORM TO SET TO PREVENT DOUBLE FACTORIZATION\n",
    "    for z in set(df_num.select_dtypes(include=['object']).columns.tolist() + cat_names):\n",
    "        y, label = pd.factorize(df[z])\n",
    "        subs[z] = {'y': y, 'label': label}\n",
    "        df_num[z] = y\n",
    "    return df_num, subs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Basepath: data/Ticket/Ticket\n",
      "Source file: data/Ticket/Ticket_textual.csv\n",
      "Target file label: data/Ticket/Ticket_labels.csv\n",
      "Target file: data/Ticket/Ticket.csv\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ItinID</th>\n",
       "      <th>MktID</th>\n",
       "      <th>MktCoupons</th>\n",
       "      <th>Year</th>\n",
       "      <th>Quarter</th>\n",
       "      <th>OriginAirportID</th>\n",
       "      <th>OriginAirportSeqID</th>\n",
       "      <th>OriginCityMarketID</th>\n",
       "      <th>Origin</th>\n",
       "      <th>OriginCountry</th>\n",
       "      <th>...</th>\n",
       "      <th>OpCarrier</th>\n",
       "      <th>BulkFare</th>\n",
       "      <th>Passengers</th>\n",
       "      <th>MktFare</th>\n",
       "      <th>MktDistance</th>\n",
       "      <th>MktDistanceGroup</th>\n",
       "      <th>MktMilesFlown</th>\n",
       "      <th>NonStopMiles</th>\n",
       "      <th>ItinGeoType</th>\n",
       "      <th>MktGeoType</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>201513772623</td>\n",
       "      <td>20151377262301</td>\n",
       "      <td>1</td>\n",
       "      <td>2015</td>\n",
       "      <td>1</td>\n",
       "      <td>12892</td>\n",
       "      <td>1289203</td>\n",
       "      <td>32575</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>1242.0</td>\n",
       "      <td>3</td>\n",
       "      <td>1242.0</td>\n",
       "      <td>1242.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>201513772624</td>\n",
       "      <td>20151377262401</td>\n",
       "      <td>1</td>\n",
       "      <td>2015</td>\n",
       "      <td>1</td>\n",
       "      <td>12892</td>\n",
       "      <td>1289203</td>\n",
       "      <td>32575</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>121.0</td>\n",
       "      <td>1242.0</td>\n",
       "      <td>3</td>\n",
       "      <td>1242.0</td>\n",
       "      <td>1242.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>201513772625</td>\n",
       "      <td>20151377262501</td>\n",
       "      <td>1</td>\n",
       "      <td>2015</td>\n",
       "      <td>1</td>\n",
       "      <td>12892</td>\n",
       "      <td>1289203</td>\n",
       "      <td>32575</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>126.0</td>\n",
       "      <td>1242.0</td>\n",
       "      <td>3</td>\n",
       "      <td>1242.0</td>\n",
       "      <td>1242.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>201513772626</td>\n",
       "      <td>20151377262601</td>\n",
       "      <td>1</td>\n",
       "      <td>2015</td>\n",
       "      <td>1</td>\n",
       "      <td>12892</td>\n",
       "      <td>1289203</td>\n",
       "      <td>32575</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>129.0</td>\n",
       "      <td>1242.0</td>\n",
       "      <td>3</td>\n",
       "      <td>1242.0</td>\n",
       "      <td>1242.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>201513772627</td>\n",
       "      <td>20151377262701</td>\n",
       "      <td>1</td>\n",
       "      <td>2015</td>\n",
       "      <td>1</td>\n",
       "      <td>12892</td>\n",
       "      <td>1289203</td>\n",
       "      <td>32575</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>131.0</td>\n",
       "      <td>1242.0</td>\n",
       "      <td>3</td>\n",
       "      <td>1242.0</td>\n",
       "      <td>1242.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 40 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         ItinID           MktID  MktCoupons  Year  Quarter  OriginAirportID  \\\n",
       "0  201513772623  20151377262301           1  2015        1            12892   \n",
       "1  201513772624  20151377262401           1  2015        1            12892   \n",
       "2  201513772625  20151377262501           1  2015        1            12892   \n",
       "3  201513772626  20151377262601           1  2015        1            12892   \n",
       "4  201513772627  20151377262701           1  2015        1            12892   \n",
       "\n",
       "   OriginAirportSeqID  OriginCityMarketID  Origin  OriginCountry     ...      \\\n",
       "0             1289203               32575       0              0     ...       \n",
       "1             1289203               32575       0              0     ...       \n",
       "2             1289203               32575       0              0     ...       \n",
       "3             1289203               32575       0              0     ...       \n",
       "4             1289203               32575       0              0     ...       \n",
       "\n",
       "   OpCarrier  BulkFare  Passengers  MktFare  MktDistance  MktDistanceGroup  \\\n",
       "0          0       0.0        12.0    120.0       1242.0                 3   \n",
       "1          0       0.0         1.0    121.0       1242.0                 3   \n",
       "2          0       0.0         1.0    126.0       1242.0                 3   \n",
       "3          0       0.0        25.0    129.0       1242.0                 3   \n",
       "4          0       0.0         3.0    131.0       1242.0                 3   \n",
       "\n",
       "   MktMilesFlown  NonStopMiles  ItinGeoType  MktGeoType  \n",
       "0         1242.0        1242.0            2           2  \n",
       "1         1242.0        1242.0            2           2  \n",
       "2         1242.0        1242.0            2           2  \n",
       "3         1242.0        1242.0            2           2  \n",
       "4         1242.0        1242.0            2           2  \n",
       "\n",
       "[5 rows x 40 columns]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a, b, c = transform('Ticket', suffix='_textual', to_disk=True, target='OpCarrierGroup')\n",
    "a.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}