Property_type_classifier.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/pouya/anaconda3/envs/python38/lib/python3.8/site-packages/IPython/core/interactiveshell.py:3062: DtypeWarning: Columns (3,8,9,16,18,19,21,22,26) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  has_raised = await self.run_ast_nodes(code_ast.body, cell_name,\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import sklearn.cluster as cluster\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.preprocessing import StandardScaler as sc\n",
    "import hdbscan\n",
    "\n",
    "big_data = pd.read_csv('airbnb_coded.csv')\n",
    "sc = sc()\n",
    "pd.options.mode.chained_assignment = None  # default='warn'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data['property_types'] = data.property_type.map({'Apartment':0, 'House':1, 'Loft':2, 'Hotel':3, 'Condominium':4, 'Tiny house':5, 'Serviced apartment':6, 'Boutique hotel':7,\n",
    "#                                            'Guest suit':8, 'Houseboat':9, 'Hostel':10, 'Villa':11, 'Townhouse':12, 'Cabin':13, 'Camper/RV':14})\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "\n",
    "ct = ColumnTransformer(transformers=[('encoder', OneHotEncoder(), [0])], remainder='passthrough')\n",
    "var = 'bathrooms'\n",
    "var2 = 'accommodates'\n",
    "var3 = 'cleaning_fee'\n",
    "var10 = 'price_x'\n",
    "\n",
    "data = big_data[['property_type', var, var2, var3, var10]]#, 'cleaning_fee', 'security_deposit']]\n",
    "# data['cleaning_fee'] = data['cleaning_fee'].fillna(data['cleaning_fee'].median())\n",
    "\n",
    "\n",
    "data[var3] = data[var3].str.replace(',', '')\n",
    "data[var3] = data[var3].str.replace('$', '')\n",
    "\n",
    "big_data[var3] = big_data[var3].str.replace(',', '')\n",
    "big_data[var3] = big_data[var3].str.replace('$', '')\n",
    "\n",
    "data[var10] = data[var10].str.replace(',', '')\n",
    "data[var10] = data[var10].str.replace('$', '')\n",
    "\n",
    "big_data[var10] = big_data[var10].str.replace(',', '')\n",
    "big_data[var10] = big_data[var10].str.replace('$', '')\n",
    "\n",
    "\n",
    "data[var][6722] = 0\n",
    "data[var][4476] = 0\n",
    "data[var2][6722] = 0\n",
    "data[var2][4476] = 0\n",
    "data[var3][6722] = 0\n",
    "data[var3][4476] = 0\n",
    "# data['security_deposit'][4476] = 0\n",
    "# data['security_deposit'] = data['security_deposit'].fillna(data['security_deposit'].median())\n",
    "data[var] = data[var].fillna(data[var].median())\n",
    "data[var2] = data[var2].fillna(data[var2].median())\n",
    "data[var3] = data[var3].fillna(data[var3].median())\n",
    "data[var10] = data[var10].fillna(data[var10].median())\n",
    "data['property_type'] = data['property_type'].fillna('Apartment')\n",
    "# data.head()\n",
    "data = data.drop(data.index[14973])\n",
    "big_data = big_data.drop(data.index[14973])\n",
    "big_data = big_data.drop(big_data.index[6722])\n",
    "big_data = big_data.drop(big_data.index[4476])\n",
    "data = data.drop(data.index[6722])\n",
    "data = data.drop(data.index[4476])\n",
    "X = data.iloc[:, 1:-1].values\n",
    "# X = ct.fit_transform(X)\n",
    "# X = X.todense()\n",
    "# X = np.array(X)\n",
    "# X1 = data[var2].values #, data['cleaning_fee'].values, data['security_deposit'].values\n",
    "# # X2 = \n",
    "# y = data['property_type'].values\n",
    "# cluster_data = []\n",
    "# for i in range(len(X1)):\n",
    "#     cluster_data.append([y[i], X1[i]])\n",
    "# cluster_data = np.array(cluster_data)\n",
    "# # cluster_data = sc.fit_transform(cluster_data)\n",
    "# print(len(cluster_data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# label = hdbscan.HDBSCAN(min_cluster_size=450, min_samples=15).fit_predict(cluster_data)\n",
    "label = cluster.KMeans(n_clusters=10).fit_predict(X)\n",
    "max(label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>property_type</th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>accommodates</th>\n",
       "      <th>cleaning_fee</th>\n",
       "      <th>price_x</th>\n",
       "      <th>listing_class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Apartment</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>30.00</td>\n",
       "      <td>35.00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Apartment</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4</td>\n",
       "      <td>20</td>\n",
       "      <td>80.00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Apartment</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4</td>\n",
       "      <td>5.00</td>\n",
       "      <td>50.00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Apartment</td>\n",
       "      <td>1.5</td>\n",
       "      <td>6</td>\n",
       "      <td>80.00</td>\n",
       "      <td>80.00</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Apartment</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5</td>\n",
       "      <td>50.00</td>\n",
       "      <td>180.00</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18039</th>\n",
       "      <td>Apartment</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>20.00</td>\n",
       "      <td>90.00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18040</th>\n",
       "      <td>Apartment</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>30.00</td>\n",
       "      <td>95.00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18041</th>\n",
       "      <td>Apartment</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>10.00</td>\n",
       "      <td>39.00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18042</th>\n",
       "      <td>Condominium</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>10.00</td>\n",
       "      <td>53.00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18043</th>\n",
       "      <td>Apartment</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>20</td>\n",
       "      <td>13.00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>18041 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      property_type bathrooms accommodates cleaning_fee price_x  listing_class\n",
       "0         Apartment       1.0            2        30.00   35.00              0\n",
       "1         Apartment       1.0            4           20   80.00              0\n",
       "2         Apartment       1.0            4         5.00   50.00              0\n",
       "3         Apartment       1.5            6        80.00   80.00              3\n",
       "4         Apartment       1.0            5        50.00  180.00              1\n",
       "...             ...       ...          ...          ...     ...            ...\n",
       "18039     Apartment         1            3        20.00   90.00              0\n",
       "18040     Apartment         1            5        30.00   95.00              0\n",
       "18041     Apartment         1            2        10.00   39.00              0\n",
       "18042   Condominium         1            2        10.00   53.00              0\n",
       "18043     Apartment         1            1           20   13.00              0\n",
       "\n",
       "[18041 rows x 6 columns]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# label = hdbscan.HDBSCAN(min_cluster_size=450, min_samples=15).fit_predict(cluster_data)\n",
    "clusterer = cluster.KMeans(n_clusters=4)\n",
    "X = clusterer.fit_predict(X)\n",
    "# clusterer.fit(X)\n",
    "# Clusters = clusterer.predict(X)\n",
    "data['listing_class'] = X\n",
    "big_data['listing_class'] = X\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['1.0' '2' '30.00']\n",
      " ['1.0' '4' 20.0]\n",
      " ['1.0' '4' '5.00']\n",
      " ...\n",
      " [1.0 2 '10.00']\n",
      " [1.0 2 '10.00']\n",
      " [1.0 1 20.0]]\n"
     ]
    }
   ],
   "source": [
    "data = data[['property_type', var, var2, var3, 'listing_class', var10]]\n",
    "X = data.iloc[:, 1:-2].values\n",
    "print(X)\n",
    "y = data.iloc[:, -2].values\n",
    "# X = ct.fit_transform(X)\n",
    "# X = X.todense()\n",
    "# X = np.array(X)\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.998653785239151\n",
      "0.9996305191206355\n"
     ]
    }
   ],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "classifier = DecisionTreeClassifier(max_depth=2)\n",
    "classifier = classifier.fit(X_train, y_train)\n",
    "print(classifier.score(X_train, y_train))\n",
    "print(classifier.score(X_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.0\n",
      "1.0\n"
     ]
    }
   ],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "classifier = KNeighborsClassifier(n_neighbors=65, weights='distance')\n",
    "classifier = classifier.fit(X_train, y_train)\n",
    "print(classifier.score(X_train, y_train))\n",
    "print(classifier.score(X_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "classifier = RandomForestClassifier(n_estimators=500, min_samples_leaf=1)\n",
    "classifier = classifier.fit(X_train, y_train)\n",
    "print(classifier.score(X_train, y_train))\n",
    "print(classifier.score(X_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Lets now use cross validation:\n",
    "from sklearn.model_selection import cross_val_score\n",
    "scores = cross_val_score(classifier, X_train, \n",
    "                          y_train, scoring='accuracy', cv=10)\n",
    "print(scores)\n",
    "np.mean(scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# After clustering of property types\n",
    "big_data.to_csv('airbnb_coded.2.0.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestRegressor as RFR\n",
    "rf = RFR(n_estimators=500, min_samples_leaf=1)\n",
    "rf = rf.fit(X_train, y_train)\n",
    "rf.score(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rf.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}