get-started-with-mean-imputation-all-zero.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "6b368fa4",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2022-06-10T01:54:10.965438Z",
     "iopub.status.busy": "2022-06-10T01:54:10.964953Z",
     "iopub.status.idle": "2022-06-10T01:54:12.226490Z",
     "shell.execute_reply": "2022-06-10T01:54:12.225430Z"
    },
    "papermill": {
     "duration": 1.270253,
     "end_time": "2022-06-10T01:54:12.229367",
     "exception": false,
     "start_time": "2022-06-10T01:54:10.959114",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from pathlib import Path\n",
    "from tqdm import tqdm\n",
    "\n",
    "from sklearn.impute import SimpleImputer"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7b8922db",
   "metadata": {
    "papermill": {
     "duration": 0.002556,
     "end_time": "2022-06-10T01:54:12.235460",
     "exception": false,
     "start_time": "2022-06-10T01:54:12.232904",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Getting Started: Exploring Different Imputation Methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "40ff7239",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-10T01:54:12.242927Z",
     "iopub.status.busy": "2022-06-10T01:54:12.242505Z",
     "iopub.status.idle": "2022-06-10T01:54:31.590528Z",
     "shell.execute_reply": "2022-06-10T01:54:31.589563Z"
    },
    "papermill": {
     "duration": 19.354835,
     "end_time": "2022-06-10T01:54:31.593168",
     "exception": false,
     "start_time": "2022-06-10T01:54:12.238333",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "input_path = Path('/kaggle/input/tabular-playground-series-jun-2022/')\n",
    "\n",
    "data = pd.read_csv(input_path / 'data.csv', index_col='row_id')\n",
    "submission = pd.read_csv(input_path / 'sample_submission.csv', index_col='row-col')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1013e370",
   "metadata": {
    "papermill": {
     "duration": 0.004095,
     "end_time": "2022-06-10T01:54:31.600796",
     "exception": false,
     "start_time": "2022-06-10T01:54:31.596701",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Use scikit-learn SimpleImputer for a simple benchmark\n",
    "\n",
    "There are many ways to impute missing values in data. One of the easiest is to simply replace the missing values of a feature with the mean of the feature values. `scikit-learn` has a method for this: `SimpleImputer`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "6ae00e2e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-10T01:54:31.611179Z",
     "iopub.status.busy": "2022-06-10T01:54:31.610404Z",
     "iopub.status.idle": "2022-06-10T01:54:36.644943Z",
     "shell.execute_reply": "2022-06-10T01:54:36.643825Z"
    },
    "papermill": {
     "duration": 5.043886,
     "end_time": "2022-06-10T01:54:36.647516",
     "exception": false,
     "start_time": "2022-06-10T01:54:31.603630",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "F = [[], [], [], [], []]\n",
    "features = list(data.columns)\n",
    "\n",
    "for feature in features:\n",
    "    for i in [1, 2, 3, 4]:\n",
    "        if feature.split('_')[1] == str(i):\n",
    "            F[i].append(feature)\n",
    "df = [[], [], [], [], []]\n",
    "\n",
    "for i in [1, 2, 3, 4]:\n",
    "    df[i] = data[F[i]]\n",
    "    corr = df[i].corr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ef9f20f8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-10T01:54:36.656035Z",
     "iopub.status.busy": "2022-06-10T01:54:36.655193Z",
     "iopub.status.idle": "2022-06-10T01:54:37.853461Z",
     "shell.execute_reply": "2022-06-10T01:54:37.852189Z"
    },
    "papermill": {
     "duration": 1.205722,
     "end_time": "2022-06-10T01:54:37.856708",
     "exception": false,
     "start_time": "2022-06-10T01:54:36.650986",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:7: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  import sys\n",
      "/opt/conda/lib/python3.7/site-packages/pandas/core/indexing.py:1732: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_block(indexer, value, name)\n",
      "/opt/conda/lib/python3.7/site-packages/pandas/core/indexing.py:723: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  iloc._setitem_with_indexer(indexer, value, self.name)\n",
      "/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:7: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  import sys\n",
      "/opt/conda/lib/python3.7/site-packages/pandas/core/indexing.py:1732: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_block(indexer, value, name)\n",
      "/opt/conda/lib/python3.7/site-packages/pandas/core/indexing.py:723: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  iloc._setitem_with_indexer(indexer, value, self.name)\n",
      "/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:7: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  import sys\n",
      "/opt/conda/lib/python3.7/site-packages/pandas/core/indexing.py:1732: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_block(indexer, value, name)\n",
      "/opt/conda/lib/python3.7/site-packages/pandas/core/indexing.py:723: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  iloc._setitem_with_indexer(indexer, value, self.name)\n"
     ]
    }
   ],
   "source": [
    "imp = SimpleImputer(\n",
    "        missing_values=np.nan,\n",
    "        strategy='constant',\n",
    "        fill_value=0)\n",
    "\n",
    "for i in [1, 3, 4]:\n",
    "    df[i][:] = imp.fit_transform(df[i])\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3fa0da47",
   "metadata": {
    "papermill": {
     "duration": 0.002998,
     "end_time": "2022-06-10T01:54:37.864114",
     "exception": false,
     "start_time": "2022-06-10T01:54:37.861116",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Use `row-col` from the sample submission to find the imputed values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "52310ae1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-10T01:54:37.872385Z",
     "iopub.status.busy": "2022-06-10T01:54:37.871876Z",
     "iopub.status.idle": "2022-06-10T01:56:52.158407Z",
     "shell.execute_reply": "2022-06-10T01:56:52.157332Z"
    },
    "papermill": {
     "duration": 134.293747,
     "end_time": "2022-06-10T01:56:52.161111",
     "exception": false,
     "start_time": "2022-06-10T01:54:37.867364",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1000000/1000000 [02:12<00:00, 7543.30it/s]\n"
     ]
    }
   ],
   "source": [
    "d = pd.concat([df[1], df[2], df[3], df[4]], axis=1)\n",
    "for i in tqdm(submission.index):\n",
    "    row = int(i.split('-')[0])\n",
    "    col = i.split('-')[1]\n",
    "    submission.loc[i, 'value'] = d.loc[row, col]\n",
    "\n",
    "submission.to_csv('zero_1.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 172.89307,
   "end_time": "2022-06-10T01:56:53.282446",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2022-06-10T01:54:00.389376",
   "version": "2.3.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}