Clean_data-tayara.tn.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import pandas_profiling\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv('Unclean_data-tayara.tn.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['location', 'category', 'chambres', 'salles_de_bains', 'superficie',\n",
       "       'type_de_transaction', 'date', 'price'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "newCols = {'chambres':'room_count','salles_de_bains':'bathroom_count','superficie':'size','type_de_transaction':'type'}\n",
    "data.rename(columns=newCols, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getByCategory(df,category):\n",
    "    rows = df.category.isin(category)\n",
    "    return (df[rows],rows)\n",
    "\n",
    "def printInfo(df):\n",
    "    print(df.info())\n",
    "    print(df.describe())\n",
    "    print(df.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "rows, cats = getByCategory(data, ['Appartements','Locations de vacances','Colocations'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 6771 entries, 2 to 20133\n",
      "Data columns (total 8 columns):\n",
      "location          6771 non-null object\n",
      "category          6771 non-null object\n",
      "room_count        6751 non-null float64\n",
      "bathroom_count    6751 non-null float64\n",
      "size              6494 non-null float64\n",
      "type              6749 non-null object\n",
      "date              6771 non-null object\n",
      "price             6768 non-null float64\n",
      "dtypes: float64(4), object(4)\n",
      "memory usage: 476.1+ KB\n",
      "None\n",
      "         room_count  bathroom_count          size         price\n",
      "count  6.751000e+03    6.751000e+03  6.494000e+03  6.768000e+03\n",
      "mean   2.032356e+05    1.715703e+05  5.407338e+05  6.325050e+05\n",
      "std    3.779753e+06    3.380941e+06  2.694343e+07  1.081339e+07\n",
      "min   -4.000000e+00   -9.000000e+00 -2.500000e+02  0.000000e+00\n",
      "25%    2.000000e+00    1.000000e+00  7.000000e+01  1.700000e+02\n",
      "50%    2.000000e+00    1.000000e+00  1.000000e+02  7.000000e+02\n",
      "75%    3.000000e+00    1.000000e+00  1.270000e+02  1.100000e+05\n",
      "max    9.869602e+07    9.869602e+07  2.147484e+09  5.700000e+08\n",
      "                location               category  room_count  bathroom_count  \\\n",
      "2  Ariana, Autres villes  Locations de vacances         3.0            80.0   \n",
      "3  Ariana, Autres villes           Appartements         2.0             1.0   \n",
      "4  Ariana, Autres villes  Locations de vacances         1.0             1.0   \n",
      "5  Ariana, Autres villes           Appartements         2.0             1.0   \n",
      "6  Ariana, Autres villes  Locations de vacances         1.0             1.0   \n",
      "\n",
      "    size     type            date  price  \n",
      "2    NaN  À Louer  7 février 2020   70.0  \n",
      "3  900.0  À Louer  7 février 2020   80.0  \n",
      "4  900.0  À Louer  7 février 2020   70.0  \n",
      "5   80.0  À Louer  7 février 2020  380.0  \n",
      "6   90.0  À Louer  7 février 2020   70.0  \n"
     ]
    }
   ],
   "source": [
    "printInfo(rows)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "rooms = data.room_count.between(1,4)\n",
    "bathrooms = data.bathroom_count.between(1,3)\n",
    "size = data['size'].between(20,500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "dropped = data[(cats & ~(rooms & bathrooms & size))]\n",
    "data.drop(dropped.index,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "rows, cat = getByCategory(data, ['Maisons et Villas'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 5020 entries, 17 to 20136\n",
      "Data columns (total 8 columns):\n",
      "location          5020 non-null object\n",
      "category          5020 non-null object\n",
      "room_count        5003 non-null float64\n",
      "bathroom_count    5003 non-null float64\n",
      "size              4786 non-null float64\n",
      "type              5002 non-null object\n",
      "date              5020 non-null object\n",
      "price             5019 non-null float64\n",
      "dtypes: float64(4), object(4)\n",
      "memory usage: 353.0+ KB\n",
      "None\n",
      "         room_count  bathroom_count          size         price\n",
      "count  5.003000e+03    5.003000e+03  4.786000e+03  5.019000e+03\n",
      "mean   9.134172e+05    1.325793e+06  1.050431e+06  3.573246e+06\n",
      "std    4.299557e+07    5.260999e+07  3.804968e+07  6.430572e+07\n",
      "min   -1.110000e+02   -2.111000e+03 -1.111100e+04  0.000000e+00\n",
      "25%    3.000000e+00    1.000000e+00  1.012500e+02  6.500000e+01\n",
      "50%    3.000000e+00    1.000000e+00  1.850000e+02  6.500000e+02\n",
      "75%    4.000000e+00    2.000000e+00  3.500000e+02  2.100000e+05\n",
      "max    2.147484e+09    2.147484e+09  2.147484e+09  4.000000e+09\n",
      "                 location           category  room_count  bathroom_count  \\\n",
      "17      Ariana, La Soukra  Maisons et Villas         4.0             2.0   \n",
      "19       Ariana, Chotrana  Maisons et Villas         5.0             3.0   \n",
      "21         Ariana, Raoued  Maisons et Villas         2.0             1.0   \n",
      "31  Ariana, Autres villes  Maisons et Villas         4.0             2.0   \n",
      "33        Ariana, Ghazela  Maisons et Villas         2.0             1.0   \n",
      "\n",
      "      size      type            date     price  \n",
      "17  1208.0  À Vendre  7 février 2020  700000.0  \n",
      "19   220.0   À Louer  7 février 2020    2000.0  \n",
      "21   150.0  À Vendre  7 février 2020  120000.0  \n",
      "31   130.0  À Vendre  7 février 2020  125000.0  \n",
      "33   150.0  À Vendre  7 février 2020  170000.0  \n"
     ]
    }
   ],
   "source": [
    "printInfo(rows)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "rooms = data.room_count.between(1,20)\n",
    "bathrooms = data.bathroom_count.between(1,15)\n",
    "size = data['size'].between(70,2000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "dropped = data[(cat & ~(rooms & bathrooms & size))]\n",
    "data.drop(dropped.index,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "rows, cat = getByCategory(data, ['Terrains et Fermes'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 5797 entries, 0 to 20132\n",
      "Data columns (total 8 columns):\n",
      "location          5797 non-null object\n",
      "category          5797 non-null object\n",
      "room_count        0 non-null float64\n",
      "bathroom_count    0 non-null float64\n",
      "size              0 non-null float64\n",
      "type              0 non-null object\n",
      "date              5797 non-null object\n",
      "price             5797 non-null float64\n",
      "dtypes: float64(4), object(4)\n",
      "memory usage: 407.6+ KB\n",
      "None\n",
      "       room_count  bathroom_count  size         price\n",
      "count         0.0             0.0   0.0  5.797000e+03\n",
      "mean          NaN             NaN   NaN  2.208495e+07\n",
      "std           NaN             NaN   NaN  1.321961e+09\n",
      "min           NaN             NaN   NaN  0.000000e+00\n",
      "25%           NaN             NaN   NaN  0.000000e+00\n",
      "50%           NaN             NaN   NaN  1.200000e+02\n",
      "75%           NaN             NaN   NaN  6.000000e+04\n",
      "max           NaN             NaN   NaN  1.000000e+11\n",
      "                 location            category  room_count  bathroom_count  \\\n",
      "0          Ariana, Raoued  Terrains et Fermes         NaN             NaN   \n",
      "1   Ariana, Autres villes  Terrains et Fermes         NaN             NaN   \n",
      "14  Ariana, Autres villes  Terrains et Fermes         NaN             NaN   \n",
      "18      Ariana, La Soukra  Terrains et Fermes         NaN             NaN   \n",
      "26    Ariana, Sidi Thabet  Terrains et Fermes         NaN             NaN   \n",
      "\n",
      "    size type            date     price  \n",
      "0    NaN  NaN  7 février 2020     100.0  \n",
      "1    NaN  NaN  7 février 2020  316000.0  \n",
      "14   NaN  NaN  7 février 2020     180.0  \n",
      "18   NaN  NaN  7 février 2020     600.0  \n",
      "26   NaN  NaN  7 février 2020  350000.0  \n"
     ]
    }
   ],
   "source": [
    "printInfo(rows)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.loc[cat,['room_count','bathroom_count','size']] = -1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.loc[cat, 'type'] = 'À Vendre'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "rows, cats = getByCategory(data, ['Magasins, Commerces et Locaux industriels','Bureaux et Plateaux'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 2442 entries, 7 to 20131\n",
      "Data columns (total 8 columns):\n",
      "location          2442 non-null object\n",
      "category          2442 non-null object\n",
      "room_count        2346 non-null float64\n",
      "bathroom_count    2346 non-null float64\n",
      "size              1907 non-null float64\n",
      "type              2347 non-null object\n",
      "date              2442 non-null object\n",
      "price             2442 non-null float64\n",
      "dtypes: float64(4), object(4)\n",
      "memory usage: 171.7+ KB\n",
      "None\n",
      "         room_count  bathroom_count          size         price\n",
      "count  2.346000e+03    2.346000e+03  1.907000e+03  2.442000e+03\n",
      "mean   5.380286e+04    9.661215e+04  2.549329e+06  2.390520e+07\n",
      "std    1.386735e+06    2.471236e+06  6.976039e+07  1.113262e+09\n",
      "min   -3.000000e+00   -2.000000e+00 -1.500000e+03  0.000000e+00\n",
      "25%    1.000000e+00    1.000000e+00  4.000000e+01  0.000000e+00\n",
      "50%    1.000000e+00    1.000000e+00  9.000000e+01  6.500000e+02\n",
      "75%    2.000000e+00    2.000000e+00  2.000000e+02  3.200000e+03\n",
      "max    5.149324e+07    9.912254e+07  2.147484e+09  5.500000e+10\n",
      "                       location                                   category  \\\n",
      "7   Ariana, Jardins D'el Menzah  Magasins, Commerces et Locaux industriels   \n",
      "12          Ariana, Borj Louzir  Magasins, Commerces et Locaux industriels   \n",
      "15         Ariana, Ariana Ville  Magasins, Commerces et Locaux industriels   \n",
      "24        Ariana, Autres villes  Magasins, Commerces et Locaux industriels   \n",
      "28          Ariana, Borj Louzir  Magasins, Commerces et Locaux industriels   \n",
      "\n",
      "    room_count  bathroom_count  size      type            date    price  \n",
      "7          1.0             0.0  14.0  À Vendre  7 février 2020  30000.0  \n",
      "12         2.0             1.0  20.0   À Louer  7 février 2020    650.0  \n",
      "15         1.0             1.0  40.0   À Louer  7 février 2020    450.0  \n",
      "24         1.0            13.0   NaN   À Louer  7 février 2020     13.0  \n",
      "28         1.0             1.0  20.0  À Vendre  7 février 2020  55000.0  \n"
     ]
    }
   ],
   "source": [
    "printInfo(rows)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "rooms = data.room_count.between(1,15)\n",
    "bathrooms = data.bathroom_count.between(0,10)\n",
    "size = data['size'].between(20,2000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "dropped = data[(cats & ~(rooms & bathrooms & size))]\n",
    "data.drop(dropped.index,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "rows, cat = getByCategory(data, ['Autre Immobilier'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 107 entries, 466 to 20095\n",
      "Data columns (total 8 columns):\n",
      "location          107 non-null object\n",
      "category          107 non-null object\n",
      "room_count        0 non-null float64\n",
      "bathroom_count    0 non-null float64\n",
      "size              0 non-null float64\n",
      "type              0 non-null object\n",
      "date              107 non-null object\n",
      "price             107 non-null float64\n",
      "dtypes: float64(4), object(4)\n",
      "memory usage: 7.5+ KB\n",
      "None\n",
      "       room_count  bathroom_count  size         price\n",
      "count         0.0             0.0   0.0  1.070000e+02\n",
      "mean          NaN             NaN   NaN  7.959739e+06\n",
      "std           NaN             NaN   NaN  7.262881e+07\n",
      "min           NaN             NaN   NaN  0.000000e+00\n",
      "25%           NaN             NaN   NaN  0.000000e+00\n",
      "50%           NaN             NaN   NaN  2.500000e+02\n",
      "75%           NaN             NaN   NaN  3.000000e+03\n",
      "max           NaN             NaN   NaN  7.500000e+08\n",
      "                   location          category  room_count  bathroom_count  \\\n",
      "466   Ariana, Autres villes  Autre Immobilier         NaN             NaN   \n",
      "581          Ariana, Ariana  Autre Immobilier         NaN             NaN   \n",
      "718       Ariana, La Soukra  Autre Immobilier         NaN             NaN   \n",
      "1016        Ariana, Ghazela  Autre Immobilier         NaN             NaN   \n",
      "1018       Ariana, Chotrana  Autre Immobilier         NaN             NaN   \n",
      "\n",
      "      size type            date        price  \n",
      "466    NaN  NaN  5 février 2020       2000.0  \n",
      "581    NaN  NaN  5 février 2020          0.0  \n",
      "718    NaN  NaN  4 février 2020  750000000.0  \n",
      "1016   NaN  NaN  2 février 2020        200.0  \n",
      "1018   NaN  NaN  2 février 2020        200.0  \n"
     ]
    }
   ],
   "source": [
    "printInfo(rows)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.drop(rows.index,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.dropna(subset=['price'],inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "dropped = data.price < 10\n",
    "data.drop(data[dropped].index,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "forSale = data.type == 'À Vendre'\n",
    "below10k = data.price < 10000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.loc[forSale & below10k,'price'] *= 1000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.drop(columns=['date'],inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "data[['city','region']] = data.location.str.split(', ',expand=True)\n",
    "data.drop('location',axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['log_price'] = np.log10(data['price'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.to_csv(\"Clean_data-tayara.tn.csv\", index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}