final ff 2 вариант.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "uXxU165USd57"
   },
   "source": [
    "Итоговая работа\n",
    "=====================\n",
    "\n",
    "Поздравляю вас, теперь вы вооружены знаниями по предобработке данных, и пора применить их на интересном кейсе.\n",
    "\n",
    "Представлены данные такси, по которым мы должны предсказать возьмёт ли таксист заказ или нет(driver_response).\n",
    "\n",
    "Принципы вы знаете, чек-лист аналогичен лабораторной работе.\n",
    "\n",
    "Чек-лист:\n",
    "1. Загрузите датасет taxi.csv.\n",
    "2. Посмотрите на данные. Отобразите общую информацию по признакам (вспомните о describe и info). Напишите в markdown свои наблюдения.\n",
    "3. Выявите пропуски, а также возможные причины их возникновения. Решите, что следует сделать с ними. Напишите в markdown свои наблюдения.\n",
    "4. Оцените зависимости переменных между собой. Используйте корреляции. Будет хорошо, если воспользуетесь profile_report. Напишите в markdown свои наблюдения.\n",
    "5. Определите стратегию преобразования категориальных признаков (т.е. как их сделать адекватными для моделей).\n",
    "6. Найдите признаки, которые можно разделить на другие, или преобразовать в другой тип данных. Удалите лишние, при необходимости.\n",
    "7. Разделите выборку на обучаемую и тестовую.\n",
    "8. Обучите модель. Напишите в markdown свои наблюдения по полученным результатам. Хорошие результаты дают классификаторы RandomForest и XGBoost\n",
    "\n",
    "Если возникнут затруднения, то смотрите на материал практических занятий. Данного там должно хватить для выполнения всех пунктов. Желаю успеха!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "OKb3VHXySckq"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import sys\n",
    "from math import sqrt\n",
    "import math\n",
    "from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "import pandas_profiling\n",
    "from sklearn import preprocessing\n",
    "\n",
    "from matplotlib import pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 215
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 585,
     "status": "ok",
     "timestamp": 1572600618035,
     "user": {
      "displayName": "Ivan Droba",
      "photoUrl": "",
      "userId": "04075451000500905139"
     },
     "user_tz": -180
    },
    "id": "hga37xS1Stdf",
    "outputId": "0ce57c5c-1243-4531-d0b1-d44bcc8d2e8c"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>offer_gk</th>\n",
       "      <th>weekday_key</th>\n",
       "      <th>hour_key</th>\n",
       "      <th>driver_gk</th>\n",
       "      <th>order_gk</th>\n",
       "      <th>driver_latitude</th>\n",
       "      <th>driver_longitude</th>\n",
       "      <th>origin_order_latitude</th>\n",
       "      <th>origin_order_longitude</th>\n",
       "      <th>distance_km</th>\n",
       "      <th>duration_min</th>\n",
       "      <th>offer_class_group</th>\n",
       "      <th>ride_type_desc</th>\n",
       "      <th>driver_response</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1105373</td>\n",
       "      <td>5</td>\n",
       "      <td>20</td>\n",
       "      <td>6080</td>\n",
       "      <td>174182</td>\n",
       "      <td>55.818842</td>\n",
       "      <td>37.334562</td>\n",
       "      <td>55.814567</td>\n",
       "      <td>37.355010</td>\n",
       "      <td>-1.000</td>\n",
       "      <td>-1.000</td>\n",
       "      <td>Economy</td>\n",
       "      <td>private</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>759733</td>\n",
       "      <td>5</td>\n",
       "      <td>14</td>\n",
       "      <td>6080</td>\n",
       "      <td>358774</td>\n",
       "      <td>55.805342</td>\n",
       "      <td>37.515023</td>\n",
       "      <td>55.819329</td>\n",
       "      <td>37.466398</td>\n",
       "      <td>18.802</td>\n",
       "      <td>25.217</td>\n",
       "      <td>Standard</td>\n",
       "      <td>private</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>416977</td>\n",
       "      <td>6</td>\n",
       "      <td>14</td>\n",
       "      <td>6080</td>\n",
       "      <td>866260</td>\n",
       "      <td>55.813978</td>\n",
       "      <td>37.347688</td>\n",
       "      <td>55.814827</td>\n",
       "      <td>37.354074</td>\n",
       "      <td>6.747</td>\n",
       "      <td>9.800</td>\n",
       "      <td>Economy</td>\n",
       "      <td>private</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>889660</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>6080</td>\n",
       "      <td>163522</td>\n",
       "      <td>55.745922</td>\n",
       "      <td>37.421748</td>\n",
       "      <td>55.743469</td>\n",
       "      <td>37.431130</td>\n",
       "      <td>-1.000</td>\n",
       "      <td>-1.000</td>\n",
       "      <td>Economy</td>\n",
       "      <td>private</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1120055</td>\n",
       "      <td>4</td>\n",
       "      <td>16</td>\n",
       "      <td>6080</td>\n",
       "      <td>506710</td>\n",
       "      <td>55.803578</td>\n",
       "      <td>37.521602</td>\n",
       "      <td>55.812559</td>\n",
       "      <td>37.527407</td>\n",
       "      <td>12.383</td>\n",
       "      <td>19.250</td>\n",
       "      <td>Economy</td>\n",
       "      <td>private</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   offer_gk  weekday_key  hour_key  driver_gk  order_gk  driver_latitude  \\\n",
       "0   1105373            5        20       6080    174182        55.818842   \n",
       "1    759733            5        14       6080    358774        55.805342   \n",
       "2    416977            6        14       6080    866260        55.813978   \n",
       "3    889660            2         6       6080    163522        55.745922   \n",
       "4   1120055            4        16       6080    506710        55.803578   \n",
       "\n",
       "   driver_longitude  origin_order_latitude  origin_order_longitude  \\\n",
       "0         37.334562              55.814567               37.355010   \n",
       "1         37.515023              55.819329               37.466398   \n",
       "2         37.347688              55.814827               37.354074   \n",
       "3         37.421748              55.743469               37.431130   \n",
       "4         37.521602              55.812559               37.527407   \n",
       "\n",
       "   distance_km  duration_min offer_class_group ride_type_desc  driver_response  \n",
       "0       -1.000        -1.000           Economy        private                0  \n",
       "1       18.802        25.217          Standard        private                1  \n",
       "2        6.747         9.800           Economy        private                0  \n",
       "3       -1.000        -1.000           Economy        private                1  \n",
       "4       12.383        19.250           Economy        private                1  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv('taxi.csv')\n",
    "list_c = data.columns\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 336
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 613,
     "status": "ok",
     "timestamp": 1572600649236,
     "user": {
      "displayName": "Ivan Droba",
      "photoUrl": "",
      "userId": "04075451000500905139"
     },
     "user_tz": -180
    },
    "id": "5jOJSXcHTXTA",
    "outputId": "2c47a68b-7c7c-4fed-c0b9-abe8f9e67d6d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 100000 entries, 0 to 99999\n",
      "Data columns (total 14 columns):\n",
      "offer_gk                  100000 non-null int64\n",
      "weekday_key               100000 non-null int64\n",
      "hour_key                  100000 non-null int64\n",
      "driver_gk                 100000 non-null int64\n",
      "order_gk                  100000 non-null int64\n",
      "driver_latitude           100000 non-null float64\n",
      "driver_longitude          100000 non-null float64\n",
      "origin_order_latitude     100000 non-null float64\n",
      "origin_order_longitude    100000 non-null float64\n",
      "distance_km               100000 non-null float64\n",
      "duration_min              100000 non-null float64\n",
      "offer_class_group         100000 non-null object\n",
      "ride_type_desc            100000 non-null object\n",
      "driver_response           100000 non-null int64\n",
      "dtypes: float64(6), int64(6), object(2)\n",
      "memory usage: 10.7+ MB\n"
     ]
    }
   ],
   "source": [
    "data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 304
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 607,
     "status": "ok",
     "timestamp": 1572603769043,
     "user": {
      "displayName": "Ivan Droba",
      "photoUrl": "",
      "userId": "04075451000500905139"
     },
     "user_tz": -180
    },
    "id": "dcDz1tlVUSDj",
    "outputId": "59aeb2ae-4758-4a30-bd09-6c2ca14faeae"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>offer_gk</th>\n",
       "      <th>weekday_key</th>\n",
       "      <th>hour_key</th>\n",
       "      <th>driver_gk</th>\n",
       "      <th>order_gk</th>\n",
       "      <th>driver_latitude</th>\n",
       "      <th>driver_longitude</th>\n",
       "      <th>origin_order_latitude</th>\n",
       "      <th>origin_order_longitude</th>\n",
       "      <th>distance_km</th>\n",
       "      <th>duration_min</th>\n",
       "      <th>driver_response</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1.000000e+05</td>\n",
       "      <td>100000.000000</td>\n",
       "      <td>100000.000000</td>\n",
       "      <td>100000.000000</td>\n",
       "      <td>100000.00000</td>\n",
       "      <td>100000.000000</td>\n",
       "      <td>100000.000000</td>\n",
       "      <td>100000.000000</td>\n",
       "      <td>100000.000000</td>\n",
       "      <td>100000.000000</td>\n",
       "      <td>100000.000000</td>\n",
       "      <td>100000.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>5.662189e+05</td>\n",
       "      <td>3.159860</td>\n",
       "      <td>13.827790</td>\n",
       "      <td>3446.376360</td>\n",
       "      <td>460867.01291</td>\n",
       "      <td>55.700824</td>\n",
       "      <td>37.492292</td>\n",
       "      <td>55.772440</td>\n",
       "      <td>37.541637</td>\n",
       "      <td>18.393518</td>\n",
       "      <td>23.134841</td>\n",
       "      <td>0.492480</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>3.259360e+05</td>\n",
       "      <td>2.054222</td>\n",
       "      <td>7.306496</td>\n",
       "      <td>2038.738677</td>\n",
       "      <td>266064.98953</td>\n",
       "      <td>2.135903</td>\n",
       "      <td>1.454640</td>\n",
       "      <td>0.665985</td>\n",
       "      <td>0.467758</td>\n",
       "      <td>41.129296</td>\n",
       "      <td>33.890105</td>\n",
       "      <td>0.499946</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>4.000000e+00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>14.000000</td>\n",
       "      <td>18.00000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>2.855552e+05</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>1421.000000</td>\n",
       "      <td>229178.50000</td>\n",
       "      <td>55.708651</td>\n",
       "      <td>37.413497</td>\n",
       "      <td>55.708032</td>\n",
       "      <td>37.407117</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>5.656715e+05</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>16.000000</td>\n",
       "      <td>3506.500000</td>\n",
       "      <td>462420.00000</td>\n",
       "      <td>55.772904</td>\n",
       "      <td>37.533664</td>\n",
       "      <td>55.772561</td>\n",
       "      <td>37.532537</td>\n",
       "      <td>10.411500</td>\n",
       "      <td>19.700000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>8.487198e+05</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>5242.000000</td>\n",
       "      <td>690894.00000</td>\n",
       "      <td>55.960667</td>\n",
       "      <td>37.624113</td>\n",
       "      <td>55.962000</td>\n",
       "      <td>37.624358</td>\n",
       "      <td>30.665250</td>\n",
       "      <td>37.867000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.130355e+06</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>23.000000</td>\n",
       "      <td>6806.000000</td>\n",
       "      <td>920344.00000</td>\n",
       "      <td>56.174097</td>\n",
       "      <td>38.280547</td>\n",
       "      <td>56.311121</td>\n",
       "      <td>38.243608</td>\n",
       "      <td>5234.283000</td>\n",
       "      <td>3958.283000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           offer_gk    weekday_key  ...   duration_min  driver_response\n",
       "count  1.000000e+05  100000.000000  ...  100000.000000    100000.000000\n",
       "mean   5.662189e+05       3.159860  ...      23.134841         0.492480\n",
       "std    3.259360e+05       2.054222  ...      33.890105         0.499946\n",
       "min    4.000000e+00       0.000000  ...      -1.000000         0.000000\n",
       "25%    2.855552e+05       1.000000  ...      -1.000000         0.000000\n",
       "50%    5.656715e+05       3.000000  ...      19.700000         0.000000\n",
       "75%    8.487198e+05       5.000000  ...      37.867000         1.000000\n",
       "max    1.130355e+06       6.000000  ...    3958.283000         1.000000\n",
       "\n",
       "[8 rows x 12 columns]"
      ]
     },
     "execution_count": 35,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Kptbczoff0TA"
   },
   "source": [
    "offer_gk\tdriver_gk order_gk -колонки которые решаем удалить, так как это ид заказа, водителя и заказчика уникальные"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "4OJOHZGkgI9L"
   },
   "outputs": [],
   "source": [
    "data_t = data.drop(['offer_gk',\t'driver_gk', 'order_gk'], axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 683,
     "status": "ok",
     "timestamp": 1572602701173,
     "user": {
      "displayName": "Ivan Droba",
      "photoUrl": "",
      "userId": "04075451000500905139"
     },
     "user_tz": -180
    },
    "id": "u3tHrKqCbB5I",
    "outputId": "c6164f05-1198-49e8-d882-ef421c3288f4"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['private', 'business', 'affiliate', 'SMB'], dtype=object)"
      ]
     },
     "execution_count": 8,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['ride_type_desc'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 50
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 682,
     "status": "ok",
     "timestamp": 1572602771096,
     "user": {
      "displayName": "Ivan Droba",
      "photoUrl": "",
      "userId": "04075451000500905139"
     },
     "user_tz": -180
    },
    "id": "fQ_aWFbVbZEz",
    "outputId": "0c0af6eb-568e-4bcf-9aab-8b6464465a80"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Economy', 'Standard', 'XL', 'Kids', 'Delivery', 'Test', 'Premium',\n",
       "       'VIP', 'VIP+'], dtype=object)"
      ]
     },
     "execution_count": 9,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['offer_class_group'].unique() "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "S5b5kIrjggHf"
   },
   "source": [
    "Это категориальные данные которые нужно будет кодировать"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(99897, 11)"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Видим значительное количество данных с расстоянием -1. Попробуем понять, что с этим данными ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_ = data_t.loc[data_t['distance_km'] == -1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(26207, 11)"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Очень значительный объем данных чтобы их отбрасывать ... \n",
    "сделаем рассчет расстояния и времени в пути, на основании данных координат водителя и заказчика\n",
    "но перед этим отбросим строки в которых координаты отсутствуют"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_t = data_t.drop(data_.query(\"(origin_order_latitude == -1 or origin_order_longitude == -1) and distance_km == -1\").index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "убираем эти 2 строки с -1 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_t = data_t.drop(data_.query(\"(driver_latitude == -1 or driver_longitude == -1)  and distance_km == -1\").index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "#таким образом мы оставили все строки где можем рассчитать расстояние по данным широты и долготы\n",
    "def conv_la(row):\n",
    "    #print(row)\n",
    "    if row['distance_km'] == -1: \n",
    "        R=6371  # Earth's radius\n",
    "        sin1=math.sin((row['driver_latitude']-row['origin_order_latitude'])/2)\n",
    "        sin2=math.sin((row['driver_longitude']-row['origin_order_longitude'])/2)\n",
    "        row['distance_km'] = 2*R*math.asin(math.sqrt(sin1*sin1+sin2*sin2*math.cos(row['driver_latitude'])*math.cos(row['origin_order_latitude'])))\n",
    "        # print(row['distance_km'])\n",
    "    if row['duration_min'] == -1:\n",
    "        row['duration_min'] = row['distance_km'] / avr_speed\n",
    "    return row    \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "# рассчитываем среднюю скорость, чтобы по ней рассчитать время  в пути по выпадающим строкам\n",
    "data_ = data_t.loc[data_t['distance_km'] != -1]\n",
    "avr_speed = np.mean(data_['distance_km']/data_['duration_min'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_t = data_t.apply(conv_la, axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(210, 11)"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_t.loc[data_t['distance_km'] > 400 ].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Удалим строки, где расстояние получилось более 400 км как выпады, это за пределами 6 сигм"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_t = data_t.drop(data_t.loc[data_t['distance_km'] > 400].index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x13e4e94ef60>"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfYAAAFQCAYAAABNrlxPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAF+VJREFUeJzt3X9MXfX9x/HX4d45FS7jS8RsxHWW2S6ypi7syrLkSmeyijNTF4Ohralb6pK1VISlc1AUKKErks2bRe66qnFZUst+VM2yZH+4rQtjSKUNUZveOZMtC4ulGpQuwv12lnI+3z/2LUrBwr3QnnvffT7+8l4+XD7vntrnPecC13POOQEAABPygt4AAABYPoQdAABDCDsAAIYQdgAADCHsAAAYQtgBADCEsAMAYAhhBwDAEMIOAIAh4aA3kImxsYllf8z/+Z+rderU/y774waBWbITs2QnS7NItuZhltlKSiKLWscZ+/8Lh0NBb2HZMEt2YpbsZGkWydY8zJIZwg4AgCGEHQAAQwg7AACGEHYAAAwh7AAAGELYAQAwhLADAGAIYQcAwBDCDgCAIYQdAABDCDsAAIYQdgAADCHsAAAYkpNv23op3Px4/6zbR3dUBbQTAAAWjzN2AAAMIewAABhC2AEAMISwAwBgCGEHAMAQwg4AgCGEHQAAQwg7AACGEHYAAAwh7AAAGELYAQAwhLADAGAIYQcAwBDCDgCAIYQdAABDCDsAAIYQdgAADCHsAAAYQtgBADCEsAMAYAhhBwDAEMIOAIAhhB0AAEMIOwAAhhB2AAAMIewAABhC2AEAMISwAwBgCGEHAMCQRYX9tdde0+bNmyVJIyMj2rhxozZt2qT29nb5vi9JSiQSqqmp0YYNG3Ts2LG01wIAgKVbMOxPP/20Hn30Ub3//vuSpK6uLjU2Nqq3t1fOOR06dEjJZFJHjhzRwYMHFY/H1dHRkfZaAACwdAuGfcWKFerp6Zm5nUwmVVlZKUmqqqrS4OCghoeHFYvF5HmeSktLNT09rfHx8bTWAgCApQsvtKC6ulpvvvnmzG3nnDzPkyTl5+drYmJCk5OTKioqmllz7v501hYXF19wHz09PUokEpKkuro6NTQ0pDHm4pSURDL6WDbKtf1eCLNkJ2bJXpbmYZb0LRj28+XlfXCSn0qlVFhYqIKCAqVSqVn3RyKRtNYupL6+XvX19ZKksbEJjY1NpLv1CyopiVzwMZf7611MC82SS5glOzFL9rI0D7PMfYzFSPu74svLyzU0NCRJ6u/vVzQaVUVFhQYGBuT7vkZHR+X7voqLi9NaCwAAli7tM/ampia1trYqHo+rrKxM1dXVCoVCikajqq2tle/7amtrS3tt0K5v/l3QWwAAYMk855wLehPpuhiXZm5+vP+CHz+6o2rZv+bFwuWr7MQs2cnSLJKteZhl7mMsBr+gBgAAQwg7AACGEHYAAAwh7AAAGELYAQAwhLADAGAIYQcAwBDCDgCAIYQdAABDCDsAAIYQdgAADCHsAAAYQtgBADCEsAMAYAhhBwDAEMIOAIAhhB0AAEMIOwAAhhB2AAAMIewAABgSDnoDueLmx/tn3T66oyqgnQAA8NE4YwcAwBDCDgCAIYQdAABDCDsAAIYQdgAADCHsAAAYQtgBADCEsAMAYAhhBwDAEMIOAIAhhB0AAEMIOwAAhhB2AAAMIewAABhC2AEAMISwAwBgCGEHAMAQwg4AgCGEHQAAQwg7AACGEHYAAAwJZ/JJU1NTam5u1okTJ5SXl6fOzk6Fw2E1NzfL8zytWrVK7e3tysvLUyKRUF9fn8LhsFpaWrR27VqNjIzMuxYAACxNRjX985//rLNnz+qXv/yltm/frh//+Mfq6upSY2Ojent75ZzToUOHlEwmdeTIER08eFDxeFwdHR2SNO9aAACwdBmFfeXKlZqenpbv+5qcnFQ4HFYymVRlZaUkqaqqSoODgxoeHlYsFpPneSotLdX09LTGx8fnXQsAAJYuo0vxV199tU6cOKGvfe1rOnXqlPbt26ejR4/K8zxJUn5+viYmJjQ5OamioqKZzzt3v3NuztqF9PT0KJFISJLq6urU0NCQydaXTUlJJNCvv5Bs3186mCU7MUv2sjQPs6Qvo7D//Oc/VywW044dO3Ty5El985vf1NTU1MzHU6mUCgsLVVBQoFQqNev+SCQy6/X0c2sXUl9fr/r6eknS2NiExsYWfjJwMQX99S+kpCSS1ftLB7NkJ2bJXpbmYZa5j7EYGV2KLywsVCTy3y/wiU98QmfPnlV5ebmGhoYkSf39/YpGo6qoqNDAwIB839fo6Kh831dxcfG8awEAwNJldMb+rW99Sy0tLdq0aZOmpqb03e9+V2vWrFFra6vi8bjKyspUXV2tUCikaDSq2tpa+b6vtrY2SVJTU9OctQAAYOk855wLehPpuhiXZm5+vD+t9Ud3VC37HpYLl6+yE7NkJ0uzSLbmYZa5j7EY/PA4AACGEHYAAAwh7AAAGELYAQAwhLADAGAIYQcAwBDCDgCAIRn9ghrM/bn3bP65dgDA5YMzdgAADCHsAAAYQtgBADCEsAMAYAhhBwDAEMIOAIAhhB0AAEMIOwAAhhB2AAAMIewAABhC2AEAMISwAwBgCGEHAMAQwg4AgCGEHQAAQwg7AACGEHYAAAwh7AAAGELYAQAwhLADAGAIYQcAwBDCDgCAIYQdAABDCDsAAIYQdgAADCHsAAAYQtgBADCEsAMAYAhhBwDAEMIOAIAhhB0AAEMIOwAAhhB2AAAMIewAABgSzvQTn3zySf3pT3/S1NSUNm7cqMrKSjU3N8vzPK1atUrt7e3Ky8tTIpFQX1+fwuGwWlpatHbtWo2MjMy7FgAALE1GNR0aGtIrr7yiX/ziF9q/f7/eeustdXV1qbGxUb29vXLO6dChQ0omkzpy5IgOHjyoeDyujo4OSZp3LQAAWLqMwj4wMKDVq1dr+/bt2rp1q77yla8omUyqsrJSklRVVaXBwUENDw8rFovJ8zyVlpZqenpa4+Pj864FAABLl9Gl+FOnTml0dFT79u3Tm2++qW3btsk5J8/zJEn5+fmamJjQ5OSkioqKZj7v3P3zrV1IT0+PEomEJKmurk4NDQ2ZbP2iKSmJBL2FWbJtP0vBLNmJWbKXpXmYJX0Zhb2oqEhlZWW64oorVFZWpo9//ON66623Zj6eSqVUWFiogoICpVKpWfdHIpFZr6efW7uQ+vp61dfXS5LGxiY0Nrbwk4FL6frm3826fXRHVUA7+e9fnmz788kUs2QnZsleluZhlrmPsRgZXYr/4he/qL/85S9yzuntt9/W6dOn9eUvf1lDQ0OSpP7+fkWjUVVUVGhgYEC+72t0dFS+76u4uFjl5eVz1gIAgKXL6Iz91ltv1dGjR1VTUyPnnNra2nTdddeptbVV8XhcZWVlqq6uVigUUjQaVW1trXzfV1tbmySpqalpzloAALB0nnPOBb2JdF2MSzM3P96/rI/HpfjlwSzZiVmyl6V5mGXuYywGPzwOAIAhhB0AAEMIOwAAhhB2AAAMIewAABhC2AEAMISwAwBgCGEHAMAQwg4AgCGEHQAAQwg7AACGEHYAAAwh7AAAGELYAQAwhLADAGAIYQcAwJBw0BvA/G5+vH/W7aM7qgLaCQAgl3DGDgCAIYQdAABDuBR/kXApHQAQBM7YAQAwhLADAGAIYQcAwBDCDgCAIYQdAABDCDsAAIbw426XyEI//nb+xwEAyARn7AAAGELYAQAwhLADAGAIYQcAwBDCDgCAIYQdAABDCDsAAIYQdgAADOEX1ASEX0gDALgYOGMHAMAQzthzxEK/khYAAIkzdgAATCHsAAAYQtgBADCE19hzFK+5AwDmwxk7AACGLCns7777rtatW6d//OMfGhkZ0caNG7Vp0ya1t7fL931JUiKRUE1NjTZs2KBjx45J0keuBQAAS5Nx2KemptTW1qYrr7xSktTV1aXGxkb19vbKOadDhw4pmUzqyJEjOnjwoOLxuDo6Oj5yLQAAWLqMw97d3a0NGzbo2muvlSQlk0lVVlZKkqqqqjQ4OKjh4WHFYjF5nqfS0lJNT09rfHx83rUAAGDpMvrmuRdeeEHFxcW65ZZb9NRTT0mSnHPyPE+SlJ+fr4mJCU1OTqqoqGjm887dP9/ahfT09CiRSEiS6urq1NDQkMnWzSopiVzwdi5jluzELNnL0jzMkr6Mwv7888/L8zwdPnxYr7/+upqamjQ+Pj7z8VQqpcLCQhUUFCiVSs26PxKJKC8vb87ahdTX16u+vl6SNDY2obGxhZ8MXE4+/OdRUhIx8+fDLNmJWbKXpXmYZe5jLEZGl+IPHDigZ599Vvv379eNN96o7u5uVVVVaWhoSJLU39+vaDSqiooKDQwMyPd9jY6Oyvd9FRcXq7y8fM5aAACwdMv2c+xNTU1qbW1VPB5XWVmZqqurFQqFFI1GVVtbK9/31dbW9pFrAQDA0nnOORf0JtJ1MS7N5PrbqH74F9Rw+So7MUt2sjSLZGseZpn7GIvBL6gBAMAQwg4AgCGEHQAAQwg7AACGEHYAAAzhbVuN4G1cAQASZ+wAAJhC2AEAMISwAwBgCGEHAMAQwg4AgCF8V7xRfJc8AFyeOGMHAMAQwg4AgCGEHQAAQwg7AACGEHYAAAwh7AAAGELYAQAwhLADAGAIYQcAwBDCDgCAIYQdAABDCDsAAIYQdgAADCHsAAAYQtgBADCEsAMAYAhhBwDAEMIOAIAhhB0AAEPCQW8Al8bNj/fPun10R1VAOwEAXEycsQMAYAhhBwDAEMIOAIAhhB0AAEMIOwAAhhB2AAAMIewAABhC2AEAMISwAwBgCGEHAMCQjH6l7NTUlFpaWnTixAmdOXNG27Zt0w033KDm5mZ5nqdVq1apvb1deXl5SiQS6uvrUzgcVktLi9auXauRkZF51wIAgKXJqKa//e1vVVRUpN7eXj399NPq7OxUV1eXGhsb1dvbK+ecDh06pGQyqSNHjujgwYOKx+Pq6OiQpHnXAgCApcvojP32229XdXX1zO1QKKRkMqnKykpJUlVVlV566SWtXLlSsVhMnueptLRU09PTGh8fn3ft+vXrl2EcLNb5bwoj8cYwAGBBRmHPz8+XJE1OTuqhhx5SY2Ojuru75XnezMcnJiY0OTmpoqKiWZ83MTEh59yctQvp6elRIpGQJNXV1amhoSGTreMCSkoiQW9hXtm6r0wwS3ayNItkax5mSV/Gb9t68uRJbd++XZs2bdKdd96pH/7whzMfS6VSKiwsVEFBgVKp1Kz7I5HIrNfTz61dSH19verr6yVJY2MTGhtb+MkA0pONf6YlJZGs3FcmmCU7WZpFsjUPs8x9jMXI6DX2d955R1u2bNHDDz+smpoaSVJ5ebmGhoYkSf39/YpGo6qoqNDAwIB839fo6Kh831dxcfG8awEAwNJldMa+b98+vffee9q7d6/27t0rSXrkkUe0e/duxeNxlZWVqbq6WqFQSNFoVLW1tfJ9X21tbZKkpqYmtba2zloLAACWznPOuaA3ka6LcWlmvm8mu9xk4zfPcSkuOzFL9rI0D7PMfYzF4IfHAQAwhLADAGAIYQcAwBDCDgCAIYQdAABDCDsAAIYQdgAADCHsAAAYQtgBADCEsAMAYAhhBwDAEMIOAIAhhB0AAEMIOwAAhhB2AAAMCQe9AWSP89+TPhvfnx0AcGGcsQMAYAhn7PhInMEDQO7hjB0AAEMIOwAAhnApHovGpXkAyH6csQMAYAhhBwDAEMIOAIAhhB0AAEMIOwAAhhB2AAAM4cfdkDF+/A0Asg9hx7Ih9AAQPC7FAwBgCGEHAMAQwg4AgCG8xo6LhtfcAeDSI+y4ZAg9AFx8hB2BIfQAsPx4jR0AAEMIOwAAhnApHlmDS/MAsHSEHVnr/NAvhCcCAEDYYchCTwQIP4DLAWHHZSPdKwDzWejJAS8nAAgaYQfSkO6Tg+UOPVclACwksLD7vq9du3bpjTfe0BVXXKHdu3frM5/5TFDbAS6JhULPGT+ApQos7H/84x915swZ/epXv9Krr76qxx57TD/96U+D2g4QiIXOwJf7CsFyfEMiTz6A7BZY2IeHh3XLLbdIkr7whS/o+PHjQW0FMGup31ewmM9fju9d+LClXsW42E88FjPvpf6aPLnCLC4gLS0trq+vb+b2unXr3NTU1Eeuf+KJJ9zq1avd6tWr3RNPPLHs+7kYjxkUZslOzJKdLM3inK15mCUznnPOBfGEoqurSzfddJPuuOMOSVJVVZX6+5f3mX86Pve5z+mNN94I7OsvJ2bJTsySnSzNItmah1kyE9ivlK2oqJgJ+auvvqrVq1cHtRUAAMwI7DX29evX66WXXtKGDRvknNOePXuC2goAAGaEdu3atSuIL+x5nm699VbV1NTo3nvvVXFxcRDbmOVLX/pS0FtYNsySnZglO1maRbI1D7OkL7DX2AEAwPLjbVsBADCEsAMAYAhhBwDAEMIOAIAhhB0AAEMu67dttfIOc9/4xjcUiUQkSdddd51qa2v1gx/8QKFQSLFYTA8++GDAO1zYa6+9ph/96Efav3+/RkZG1NzcLM/ztGrVKrW3tysvL0+JREJ9fX0Kh8NqaWnR2rVrg972vD48SzKZ1NatW3X99ddLkjZu3Kg77rgj62eZmppSS0uLTpw4oTNnzmjbtm264YYbcvK4zDfLJz/5yZw8LpI0PT2tRx99VP/85z8VCoXU1dUl51xOHpv5ZpmYmMjZYyNJ7777ru655x797Gc/UzgcDua4XLJfXpuFXnzxRdfU1OScc+6VV15xW7duDXhH6fvPf/7j7r777ln33XXXXW5kZMT5vu++/e1vu+PHjwe0u8V56qmn3Ne//nV37733Ouec+853vuNefvll55xzra2t7ve//707fvy427x5s/N93504ccLdc889QW75I50/y69//Wv3zDPPzFqTC7M899xzbvfu3c4558bHx926dety9rjMN0uuHhfnnPvDH/7gmpubnXPOvfzyy27r1q05e2zmmyWXj82ZM2dcXV2du+2229zf//73wI7LZX0p3sI7zP3tb3/T6dOntWXLFt1///06evSozpw5oxUrVsjzPMViMR0+fDjobV7QihUr1NPTM3M7mUyqsrJS0n/fQ2BwcFDDw8OKxWLyPE+lpaWanp7W+Ph4UFv+SOfPcvz4cfX19em+++5TS0uLJicnc2KW22+/XQ0NDTO3Q6FQzh6X+WbJ1eMiSV/96lfV2dkpSRodHdU111yTs8dmvlly+dh0d3drw4YNuvbaayUF92/ZZR32yclJFRQUzNwOhUI6e/ZsgDtK35VXXqkHHnhAzzzzjDo6OrRz505dddVVMx/Pz8/XxMREgDtcWHV1tcLhD14Vcs7J8zxJH+z//GOVrXOdP8vatWv1/e9/XwcOHNCnP/1p/eQnP8mJWfLz81VQUKDJyUk99NBDamxszNnjMt8suXpczgmHw2pqalJnZ6eqq6tz9thIc2fJ1WPzwgsvqLi4eOZkUQru37LLOuwFBQVKpVIzt33fn/WPci5YuXKl7rrrLnmep5UrVyoSiejf//73zMdTqZQKCwsD3GH68vI++Gt5bv/nH6tUKjXzfQXZbP369VqzZs3Mf//1r3/NmVlOnjyp+++/X3fffbfuvPPOnD4u58+Sy8flnO7ubr344otqbW3V+++/P3N/rh0bafYssVgsJ4/N888/r8HBQW3evFmvv/66mpqaZp2JX8rjclmH3cI7zD333HN67LHHJElvv/22Tp8+rauvvlr/+te/5JzTwMCAotFowLtMT3l5uYaGhiRJ/f39ikajqqio0MDAgHzf1+joqHzfz4r3F1jIAw88oGPHjkmSDh8+rM9//vM5Mcs777yjLVu26OGHH1ZNTY2k3D0u882Sq8dFkn7zm9/oySeflCRdddVV8jxPa9asycljM98sDz74YE4emwMHDujZZ5/V/v37deONN6q7u1tVVVWBHJfcOj1dZhbeYa6mpkY7d+7Uxo0b5Xme9uzZo7y8PH3ve9/T9PS0YrGYbrrppqC3mZampia1trYqHo+rrKxM1dXVCoVCikajqq2tle/7amtrC3qbi7Jr1y51dnbqYx/7mK655hp1dnaqoKAg62fZt2+f3nvvPe3du1d79+6VJD3yyCPavXt3zh2X+WZpbm7Wnj17cu64SNJtt92mnTt36r777tPZs2fV0tKiz372szn5/8x8s3zqU5/Kyf9n5hPUv2W8CQwAAIZc1pfiAQCwhrADAGAIYQcAwBDCDgCAIYQdAABDCDsAAIYQdgAADCHsAAAY8n+UMdL/TRTpPQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 576x396 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data_t['distance_km'].hist(bins = 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_t = data_t.drop(['driver_latitude',\t'driver_longitude', 'origin_order_latitude','origin_order_longitude'], axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>weekday_key</th>\n",
       "      <th>hour_key</th>\n",
       "      <th>distance_km</th>\n",
       "      <th>duration_min</th>\n",
       "      <th>offer_class_group</th>\n",
       "      <th>ride_type_desc</th>\n",
       "      <th>driver_response</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5</td>\n",
       "      <td>20</td>\n",
       "      <td>100.659966</td>\n",
       "      <td>147.898997</td>\n",
       "      <td>Economy</td>\n",
       "      <td>private</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5</td>\n",
       "      <td>14</td>\n",
       "      <td>18.802000</td>\n",
       "      <td>25.217000</td>\n",
       "      <td>Standard</td>\n",
       "      <td>private</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6</td>\n",
       "      <td>14</td>\n",
       "      <td>6.747000</td>\n",
       "      <td>9.800000</td>\n",
       "      <td>Economy</td>\n",
       "      <td>private</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>44.320079</td>\n",
       "      <td>65.119188</td>\n",
       "      <td>Economy</td>\n",
       "      <td>private</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>16</td>\n",
       "      <td>12.383000</td>\n",
       "      <td>19.250000</td>\n",
       "      <td>Economy</td>\n",
       "      <td>private</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  weekday_key hour_key  distance_km  duration_min offer_class_group  \\\n",
       "0           5       20   100.659966    147.898997           Economy   \n",
       "1           5       14    18.802000     25.217000          Standard   \n",
       "2           6       14     6.747000      9.800000           Economy   \n",
       "3           2        6    44.320079     65.119188           Economy   \n",
       "4           4       16    12.383000     19.250000           Economy   \n",
       "\n",
       "  ride_type_desc driver_response  \n",
       "0        private               0  \n",
       "1        private               1  \n",
       "2        private               0  \n",
       "3        private               1  \n",
       "4        private               1  "
      ]
     },
     "execution_count": 152,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_t.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Функция переводит категорийные значения в набор колонок с 0,1\n",
    "def oh_encoder(df, list_col):\n",
    "  \n",
    "  r_df = df.copy()\n",
    "\n",
    "  for col in list_col:\n",
    "    conv = df[col].get_values().reshape(-1, 1)\n",
    "\n",
    "    # One-hot encoding\n",
    "    oh_encoder = preprocessing.OneHotEncoder()\n",
    "    oh_encoder.fit(conv)\n",
    "    oh_result = oh_encoder.transform(conv).toarray()\n",
    "\n",
    "    conv_columns = ['{}_{}'.format(col,i) for i in range(oh_result.shape[1])]\n",
    "    conv_df = pd.DataFrame(oh_result, columns=conv_columns)\n",
    "    conv_df.index = df.index\n",
    "    r_df = pd.concat([r_df, conv_df], axis=1)\n",
    "    r_df = r_df.drop([col], axis=1)\n",
    "\n",
    "  return r_df "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [],
   "source": [
    "# колонки которые будем переводить в категорийные через OneHotEncording\n",
    "list_col = 'weekday_key', 'hour_key', 'offer_class_group', 'ride_type_desc'\n",
    "data_t['weekday_key'] =  data_t['weekday_key'].astype(str)\n",
    "data_t['hour_key'] =  data_t['hour_key'].astype(str)\n",
    "#data_t['driver_response'] =  data_t['driver_response'].astype(str)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 99790 entries, 0 to 99999\n",
      "Data columns (total 7 columns):\n",
      "weekday_key          99790 non-null object\n",
      "hour_key             99790 non-null object\n",
      "distance_km          99790 non-null float64\n",
      "duration_min         99790 non-null float64\n",
      "offer_class_group    99790 non-null object\n",
      "ride_type_desc       99790 non-null object\n",
      "driver_response      99790 non-null object\n",
      "dtypes: float64(2), object(5)\n",
      "memory usage: 6.1+ MB\n"
     ]
    }
   ],
   "source": [
    "data_t.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\preprocessing\\_encoders.py:371: FutureWarning: The handling of integer data will change in version 0.22. Currently, the categories are determined based on the range [0, max(values)], while in the future they will be determined based on the unique values.\n",
      "If you want the future behaviour and silence this warning, you can specify \"categories='auto'\".\n",
      "In case you used a LabelEncoder before this OneHotEncoder to convert the categories to integers, then you can now use the OneHotEncoder directly.\n",
      "  warnings.warn(msg, FutureWarning)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\preprocessing\\_encoders.py:371: FutureWarning: The handling of integer data will change in version 0.22. Currently, the categories are determined based on the range [0, max(values)], while in the future they will be determined based on the unique values.\n",
      "If you want the future behaviour and silence this warning, you can specify \"categories='auto'\".\n",
      "In case you used a LabelEncoder before this OneHotEncoder to convert the categories to integers, then you can now use the OneHotEncoder directly.\n",
      "  warnings.warn(msg, FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "data_n = oh_encoder(data_t, list_col)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_n = data_n.drop_duplicates()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "убирается 15 тыс строк дубликатов !!!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(83107, 47)"
      ]
     },
     "execution_count": 157,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_n.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Теперь можно проводить обучение"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [],
   "source": [
    "list_res = ['driver_response']\n",
    "train_X = data_n.drop(list_res, axis = 1)\n",
    "train_Y = data_n[list_res]\n",
    "\n",
    "X_train,X_test,y_train,y_test = train_test_split(train_X,\n",
    "                                                 train_Y,\n",
    "                                                 test_size = 0.2,#%80 train, 20% test\n",
    "                                                 random_state = 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:8: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples,), for example using ravel().\n",
      "  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Точность предсказания модели:  0.6041992540007219\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "# создаем модель деревья решений\n",
    "model=RandomForestClassifier(n_estimators=100)\n",
    "\n",
    "#from sklearn.linear_model import LogisticRegression\n",
    "#from sklearn.metrics import accuracy_score \n",
    "#model = LogisticRegression()\n",
    "model.fit(X_train, y_train)\n",
    "predicted_classes = model.predict(X_test)\n",
    "accuracy = accuracy_score(y_test,predicted_classes)\n",
    "print('Точность предсказания модели: ', accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [],
   "source": [
    "list_res = ['driver_response']\n",
    "train_X = data_n.drop(list_res, axis = 1)\n",
    "train_Y = data_n[list_res]\n",
    "\n",
    "X_train,X_test,y_train,y_test = train_test_split(train_X,\n",
    "                                                 train_Y,\n",
    "                                                 test_size = 0.2,#%80 train, 20% test\n",
    "                                                 random_state = 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\preprocessing\\label.py:219: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
      "  y = column_or_1d(y, warn=True)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\preprocessing\\label.py:252: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
      "  y = column_or_1d(y, warn=True)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Точность предсказания модели:  0.6643604861027553\n"
     ]
    }
   ],
   "source": [
    "import xgboost\n",
    "\n",
    "model=xgboost.XGBClassifier()\n",
    "\n",
    "model.fit(X_train, y_train)\n",
    "predicted_classes = model.predict(X_test)\n",
    "accuracy = accuracy_score(y_test , predicted_classes)\n",
    "print('Точность предсказания модели: ', accuracy)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "Untitled7.ipynb",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}