tf-idf.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import gensim\n",
    "import matplotlib.pyplot as plt\n",
    "import re\n",
    "from pymystem3 import Mystem\n",
    "import smart_open\n",
    "from nltk.corpus import stopwords\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.metrics import *\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv('data/contracts.csv',sep=';', header=None, names=['contract','service_id','service_name','user_request'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>contract</th>\n",
       "      <th>service_id</th>\n",
       "      <th>service_name</th>\n",
       "      <th>user_request</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>354</td>\n",
       "      <td>4</td>\n",
       "      <td>Сантехник</td>\n",
       "      <td>лалсо</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>364</td>\n",
       "      <td>16</td>\n",
       "      <td>Сборка и ремонт мебели</td>\n",
       "      <td>заказ для Александра Трушкина</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>370</td>\n",
       "      <td>13</td>\n",
       "      <td>Ремонт компьютеров</td>\n",
       "      <td>заказ для Алишера</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>379</td>\n",
       "      <td>14</td>\n",
       "      <td>Муж на час</td>\n",
       "      <td>для Ерлана</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>385</td>\n",
       "      <td>14</td>\n",
       "      <td>Муж на час</td>\n",
       "      <td>мастер мастеров</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   contract  service_id            service_name                   user_request\n",
       "0       354           4               Сантехник                          лалсо\n",
       "1       364          16  Сборка и ремонт мебели  заказ для Александра Трушкина\n",
       "2       370          13      Ремонт компьютеров              заказ для Алишера\n",
       "3       379          14              Муж на час                     для Ерлана\n",
       "4       385          14              Муж на час                мастер мастеров"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7f9fb4a65ac8>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD/CAYAAAD4xAEfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAF1JJREFUeJzt3X+0XWV95/H3NwnQKgpBroAQDK2hFtsRNQVmwCUdbAy4lmDHccC1JDI6cWZAdIb5EbUzcbQ4zCzF1lVlLawRsCJlKpa0RiFFWmpnwARIAyEiaQRJCiQtFujo6gh854/93JWdwzn37HPvyb2B5/1aa6+zz3Oe/ezn7F+fc/be597ITCRJ9Zk31x2QJM0NA0CSKmUASFKlDABJqpQBIEmVMgAkqVIGgCRVygCQpEoZAJJUKQNAkiq1YK47MJXDDz88Fy9ePNfdkKTnlTvvvPNvMnNiWL39OgAWL17Mxo0b57obkvS8EhEPdannKSBJqpQBIEmVMgAkqVIGgCRVygCQpEoZAJJUKQNAkiplAEhSpfbrH4K1LV71jb2eP3jZW+eoJ5L0wuA3AEmqlAEgSZUyACSpUgaAJFXKAJCkShkAklSpoQEQEYsi4taIuC8itkTEB0v5xyJiZ0RsKsNZrWk+HBHbIuL+iHhLq3x5KdsWEav2zVuSJHXR5XcATwOXZOZdEfES4M6IWF9e+0xmfqpdOSJOAM4FXgO8AviTiDi+vPw54NeAHcCGiFibmfeN441IkkYzNAAy8xHgkTL+VERsBY6eYpKzgesy8x+AH0TENuCk8tq2zNwOEBHXlboGgCTNgZGuAUTEYuB1wB2l6KKI2BwRayJiYSk7Gni4NdmOUjaoXJI0Bzr/KYiIOBj4GvChzHwyIq4APgFkefw08C9n2qGIWAmsBDj22GNHmrb3z0WAfzJCkgbp9A0gIg6gOfh/JTNvAMjMxzLzmcx8FvgCe07z7AQWtSY/ppQNKt9LZl6ZmUszc+nExNB/ai9JmqYudwEF8EVga2Ze3io/qlXt7cC9ZXwtcG5EHBQRxwFLgO8CG4AlEXFcRBxIc6F47XjehiRpVF1OAZ0KvBu4JyI2lbKPAOdFxIk0p4AeBN4PkJlbIuJ6mou7TwMXZuYzABFxEXATMB9Yk5lbxvheJEkj6HIX0HeA6PPSuimmuRS4tE/5uqmmkyTNHn8JLEmVMgAkqVIGgCRVygCQpEoZAJJUKQNAkiplAEhSpQwASaqUASBJlTIAJKlSnf8c9AuFfzJakhp+A5CkShkAklQpA0CSKmUASFKlDABJqpQBIEmVMgAkqVIGgCRVygCQpEoZAJJUKQNAkiplAEhSpQwASaqUASBJlTIAJKlSBoAkVcoAkKRKGQCSVCkDQJIqZQBIUqWGBkBELIqIWyPivojYEhEfLOWHRcT6iHigPC4s5RERn42IbRGxOSJe32prRan/QESs2HdvS5I0TJdvAE8Dl2TmCcApwIURcQKwCrglM5cAt5TnAGcCS8qwErgCmsAAVgMnAycBqydDQ5I0+4YGQGY+kpl3lfGngK3A0cDZwNWl2tXAOWX8bOCabNwOHBoRRwFvAdZn5uOZ+SNgPbB8rO9GktTZSNcAImIx8DrgDuCIzHykvPQocEQZPxp4uDXZjlI2qFySNAc6B0BEHAx8DfhQZj7Zfi0zE8hxdCgiVkbExojYuHv37nE0KUnqo1MARMQBNAf/r2TmDaX4sXJqh/K4q5TvBBa1Jj+mlA0q30tmXpmZSzNz6cTExCjvRZI0gi53AQXwRWBrZl7eemktMHknzwrgxlb5+eVuoFOAJ8qpopuAZRGxsFz8XVbKJElzYEGHOqcC7wbuiYhNpewjwGXA9RHxXuAh4J3ltXXAWcA24MfABQCZ+XhEfALYUOp9PDMfH8u7kCSNbGgAZOZ3gBjw8hl96idw4YC21gBrRumgJGnf8JfAklQpA0CSKmUASFKlDABJqpQBIEmVMgAkqVIGgCRVygCQpEoZAJJUKQNAkiplAEhSpQwASaqUASBJlTIAJKlSBoAkVcoAkKRKGQCSVCkDQJIqZQBIUqUMAEmqlAEgSZUyACSpUgaAJFXKAJCkShkAklQpA0CSKmUASFKlDABJqpQBIEmVMgAkqVJDAyAi1kTEroi4t1X2sYjYGRGbynBW67UPR8S2iLg/It7SKl9eyrZFxKrxvxVJ0ii6fAO4Cljep/wzmXliGdYBRMQJwLnAa8o0n4+I+RExH/gccCZwAnBeqStJmiMLhlXIzNsiYnHH9s4GrsvMfwB+EBHbgJPKa9sycztARFxX6t43co8lSWMxk2sAF0XE5nKKaGEpOxp4uFVnRykbVC5JmiPTDYArgJ8HTgQeAT49rg5FxMqI2BgRG3fv3j2uZiVJPaYVAJn5WGY+k5nPAl9gz2mencCiVtVjStmg8n5tX5mZSzNz6cTExHS6J0nqYFoBEBFHtZ6+HZi8Q2gtcG5EHBQRxwFLgO8CG4AlEXFcRBxIc6F47fS7LUmaqaEXgSPiq8DpwOERsQNYDZweEScCCTwIvB8gM7dExPU0F3efBi7MzGdKOxcBNwHzgTWZuWXs70aS1FmXu4DO61P8xSnqXwpc2qd8HbBupN5JkvYZfwksSZUyACSpUgaAJFXKAJCkShkAklQpA0CSKmUASFKlDABJqpQBIEmVMgAkqVIGgCRVygCQpEoZAJJUKQNAkiplAEhSpQwASaqUASBJlTIAJKlSBoAkVcoAkKRKGQCSVCkDQJIqZQBIUqUMAEmqlAEgSZUyACSpUgaAJFXKAJCkShkAklQpA0CSKjU0ACJiTUTsioh7W2WHRcT6iHigPC4s5RERn42IbRGxOSJe35pmRan/QESs2DdvR5LUVZdvAFcBy3vKVgG3ZOYS4JbyHOBMYEkZVgJXQBMYwGrgZOAkYPVkaEiS5sbQAMjM24DHe4rPBq4u41cD57TKr8nG7cChEXEU8BZgfWY+npk/Atbz3FCRJM2i6V4DOCIzHynjjwJHlPGjgYdb9XaUskHlkqQ5MuOLwJmZQI6hLwBExMqI2BgRG3fv3j2uZiVJPaYbAI+VUzuUx12lfCewqFXvmFI2qPw5MvPKzFyamUsnJiam2T1J0jDTDYC1wOSdPCuAG1vl55e7gU4Bniinim4ClkXEwnLxd1kpkyTNkQXDKkTEV4HTgcMjYgfN3TyXAddHxHuBh4B3lurrgLOAbcCPgQsAMvPxiPgEsKHU+3hm9l5YliTNoqEBkJnnDXjpjD51E7hwQDtrgDUj9U6StM/4S2BJqpQBIEmVMgAkqVIGgCRVygCQpEoZAJJUKQNAkiplAEhSpQwASaqUASBJlTIAJKlSBoAkVcoAkKRKGQCSVCkDQJIqZQBIUqUMAEmqlAEgSZUyACSpUgaAJFXKAJCkShkAklQpA0CSKmUASFKlDABJqpQBIEmVMgAkqVIGgCRVygCQpEoZAJJUqRkFQEQ8GBH3RMSmiNhYyg6LiPUR8UB5XFjKIyI+GxHbImJzRLx+HG9AkjQ94/gG8KuZeWJmLi3PVwG3ZOYS4JbyHOBMYEkZVgJXjGHekqRp2hengM4Gri7jVwPntMqvycbtwKERcdQ+mL8kqYOZBkACN0fEnRGxspQdkZmPlPFHgSPK+NHAw61pd5QySdIcWDDD6U/LzJ0R8XJgfUR8r/1iZmZE5CgNliBZCXDsscfOsHuSpEFm9A0gM3eWx13A14GTgMcmT+2Ux12l+k5gUWvyY0pZb5tXZubSzFw6MTExk+5JkqYw7QCIiBdHxEsmx4FlwL3AWmBFqbYCuLGMrwXOL3cDnQI80TpVJEmaZTM5BXQE8PWImGzn2sz8VkRsAK6PiPcCDwHvLPXXAWcB24AfAxfMYN6SpBmadgBk5nbgtX3K/xY4o095AhdOd36SpPHyl8CSVCkDQJIqZQBIUqUMAEmqlAEgSZUyACSpUgaAJFXKAJCkShkAklQpA0CSKmUASFKlDABJqpQBIEmVMgAkqVIGgCRVygCQpEoZAJJUKQNAkiplAEhSpQwASaqUASBJlVow1x3YXy1e9Y29nj942VvnqCeStG/4DUCSKmUASFKlDABJqpQBIEmVMgAkqVLeBTQDvXcKwXPvFupSR5Lmgt8AJKlSfgPYT3T53YHfJiSN06wHQEQsB34bmA/8bmZeNtt9eCEzJCR1NasBEBHzgc8BvwbsADZExNrMvG82+1G7riHhr6GlF7bZ/gZwErAtM7cDRMR1wNmAAfA8Nc4L4QaONLtmOwCOBh5uPd8BnDzLfdDz2GwHjnd66YUsMnP2ZhbxDmB5Zr6vPH83cHJmXtSqsxJYWZ7+AnB/TzOHA38zZFZd6oyzLee3f/Zptue3P/bJ+b3w+9Sv3iszc2LoVJk5awPwj4GbWs8/DHx4xDY2jqPOONtyfvtnn1wGzq+GPo1Sr3eY7d8BbACWRMRxEXEgcC6wdpb7IElilq8BZObTEXERcBPNbaBrMnPLbPZBktSY9d8BZOY6YN0MmrhyTHXG2Zbz2z/7NNvz2x/75Pxmv87+Or/nmNWLwJKk/Yd/C0iSKmUASFKlXpABEBHXzHUfJGncIuLAiDg/It5cnr8rIn4nIi6MiANGbu/5fg0gInpvIw3gV4FvA2Tm22a9U31ExMszc1ef8p8Dfh1YBDwDfB+4NjOfnOUuViEiLga+npkPD628n4mIV9P8mv6OzPz7VvnyzPzW3PVM+0rvcSMivkJz886LgL8DDgZuAM6gOZ6vGGkG0/nxwFwOwMt7nt8F/B5wOvCm8vhIGX/TiG0vb40fAnwR2AxcCxzRYfpvlsfDeoaXAQ8CC4HDWvUvBm4GfgP43zR/KO9Smr+NdPoI/b6rtPHzU9T5OWAN8Jtlo/kCcC/wv4DFQ9p/2TTW01Lg1rJuFgHrgSdofgvyuhHbOgS4DPge8Djwt8DWUnZo12VQ6j0B/DXw58C/BSZG7MuV5fEftcoOKPNeC3wSeNGI6+Zg4OPAltK/3cDtwHt6tpX7gT8s29LZ7XmM2NaRwBVle3sZ8DHgHuB64KgRlsVLgf8OfBl4V89rnx8y7ffHue/1tHURcHgZfxVwG83B8g7gl0doZwHwfuBbpS+bgW8C/xo4oNSZX+p8Aji1Z/rfGKU/dDhuAJtbfXsMmF+ex+RrIy2rUSeYzaHjApkH/DuaA8yJpWx7n7Y6HURa9X+X5mD5ytL+H5by1w8Y3gA8Uuo8C/ygZ/hpedzemsc9rRX4IuBPy/ixwN1dd9bS7qeAHwLfLf19Rc/7vw34N8AqmgP/JTQH5vcC327Vu6y1sS4FtgPbgIdoBSpDDvClH2cC59H8/ad3lOnOAP5Px/U/Gag3Af8ZOLL12pGl7Oauy6DUu7tsM8toDjK7aXbwFcBLBmx37e1vR59t5dPAVTQfOj4DXNN6rcu6uRF4D3AM8O+B/wIsAa4GPtnaVg4u44uBjcAHJ9/TiG19C/hA2RY2l+W4qJTdWOoMPSADXyvbyzk04fc14KA+y+cp4MkyPFWGZybLu+575bWhoQNsaZV9A3h7GT8d+IvJ+TE8mL9Ks++dUpbnMWX8CuD3W329FvgQcCdweWv6u7r2p+txg2bfPZDmGPgUe46DPwNsHfkYO+6D9jiHLgukVfcYmk+zvwP8sE9bXQ4i7Y1wU8/0m8rjMzSnl27tM/yk1LmEZidrp/sP+vTpHvbsMAtp/ZwbuHeEnbXd7zcCnwceLX1a2ecg8cOefrRfu6c1fivwK2X8+J7+TXmAH2F+XQL1/im2kfu7LoPeeuX5AcDbaHb23a11vL1nu5t8/v/6vIdN7PlEuNcnsY7r5i97+rShPM4DvlfGt/TUObhsG5fT2lY7tjXVutnUp9+DPgz17iMfBf6CJijb038WuIbWJ3l69gc67HtlfGjotLeXyfffej75CbpLMO/1LaXfaz3regHN/fg3AAex50Pc0P5kx+NG6ed2mg9kFwO30HybvwdYPai/A9/HqBPM5tBlgfSZ5q2UTzo95V0OIjtoPjVdUhZy9Nlw7gWWDGjn4db4ZCBdDryE/t9KPkhzUP8CzTeTC0r5BHBbGe+ys97dp+35wHLgS+X5nTQH8V+h+aNRS0v5q3o2wq3AgjJ+e0+b7XCY8gBPEwLLgH9eNtZzymtvYu8g6RKoNwP/ib0PIEfQhOGflOd3DVsGg5ZV67UXlccHgGOnWsdl+/h14J/R88mL1kG4S79oTv+dVsbfxt5/L2ty2/w25Rtu67UFNAfWZ1plXdpq9+83+61jun0Y2grM63ntPTSnnx7qKX9DeQ8X04RR7we4ofvegL48J3RoTqNeRXPa8yM0n85fCVwA/HGf9zcomG+n2X7nterOA/4FzXUYKKHa06fVpU8PdO3PiMeNV1DCCjgUeAdw0qDteqphVg/o0+pghwXSsZ0uB5HVPcNEKT+S8rW+LOxfGDCPc/qUva1sSI8OmOY1pc1XD3i9y856XYf3fwbNOeStwGk0n5weAHa1+03zzeJm4J/SnG76bZqD9n8DvtyqN+UBHngtzbeubwKvLu38Hc3B4Z+02hkaqDTfjv4HTUj+iOYU3tZSdljXZVDqHd+hzoXAawe89oHy+KWeYfK0yJHALa36XdbNa2k+hf4I+M7k9kXzQeDi1n5w5IDpT52ireP7tPVxyumknnZeBfxBGe/yYeh/Am/u085yysGvp3weTQD8OfDXPa8N3ffK806hU8ruoPmw8xTNdbVPAoeU17sE82Lg92n2ke+XYVcpO67U+T1ap8tabb0P+Gnr+QVT9afP9FMeN8Y17LOGx97RGS4Q9j6IPM7eB5GFrXqvpjlYHtwz/fLyeDLw0jL+s2Vn+qPSziH92in1fqndzgj97rKzXgwsmsYy+ePenamUn1428rtpvlquo/kT3QtadYYe4IFfnGpZlvFOgVrm8eYhbU257kZYLgcB51MObsC7aE4tXkg51dPaFiZPkZ1Ac8A8a0jbp5V6y3rKf3HY+5vmdj9ofsO2864H5EHtnNnz/KTWsnoj8F/by6rrNkzH0OmZ32togqw9v64fGE4ubb0MOBX4D73ruGdeXbeDL3eY9xtprlMs69LXaW0f+6rhfdLZvQ+kF4yx3clTLx9gyJ0WNAe4yVMkVwK/VXay1cANpbzTHRtj7PfQO1tozpf2Dn8/OT7K/LrUK8vgezNZBq33N3R5dll3IyzXr9AE4B/RXGz8OvBumq/xV5c6q2k+kGykuSh5C80F19uAj7ba+m5r/F/RXC9YTXOKYFXr/c1oWQ2Y3/sGzG9Gy4oR9peuy6rLNjxCvzqtm3G006fOt/vU6bTv9dlW7u5dd+Mext7gbA30udA707bocKcFrfO9vTsLe86NdrpjY4z97nJny93M8HbZrsuc5sLajJfBiOtlbMucDrfalfnNp7l760n2/lbYPmfdvlaygT2fpF/MnlN44+z7Pp/fKOul67KiwzY8Yr+GrptxtNOxTqdb1busu3EPs/7XQEcREZsHvURz/n7cbc3L8gObzHwwIk4H/iAiXlnqAdwbERdk5peAv4yIpZm5MSKOp7lDqWs74+x3ZuazNOfuby6/CJy8Q+dTNOd/30Bz0fmjwH/MzE0R8ZPM/LNpzK9Lvae6LIMxrpexLXNgXvl/FS+m2bEPoTlleBDNXUMAT2fmM8CPI+KvsvxwLzN/EhHP9rS1kObgFpm5u9T7vxHx9D7q+4znN8b10nVZddmGu/Zr6PzG1U7HOkvpsO/Rbd2N175IlXENNJ++TqS5Yt4eFtNzEWkcbdHhTguag8FVwF/RXNT5Kc1Fsj+jXDjs0s6Y+z30zpbW82G3y3Za5sPqdV0GY1wv41zmQ2+1K+t+8q6h9l0ih7D36Y8H2XML6Xb2/HbjYPZ8Yxxn38cyv3Gtl67Lio7bcMd+dZnfuNrptB103PeGrrtxD/vs4D2WzjVfBU8b8Nq1426LjndalOcvpbkQ+gZ6fqk4Sjtj6vfQO1v6TDvodtlOy3xYva7LYFzrZZzLvEwz5a12lHvP+0x3OB1+bUrzzWLyTpKx9n0c8xvn/tJlWXXdhjv2q8v8xtXOyNvBoH2vy7ob9/C8/1tAkqTpeUH+NVBJ0nAGgCRVygCQpEoZAJJUKQNAkir1/wGnTHNHe6em2AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "data.service_id.value_counts().plot(kind='bar')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>contract</th>\n",
       "      <th>service_id</th>\n",
       "      <th>service_name</th>\n",
       "      <th>user_request</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>364</td>\n",
       "      <td>16</td>\n",
       "      <td>Сборка и ремонт мебели</td>\n",
       "      <td>заказ для Александра Трушкина</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>370</td>\n",
       "      <td>13</td>\n",
       "      <td>Ремонт компьютеров</td>\n",
       "      <td>заказ для Алишера</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>550</td>\n",
       "      <td>13</td>\n",
       "      <td>Ремонт компьютеров</td>\n",
       "      <td>где фото?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>568</td>\n",
       "      <td>13</td>\n",
       "      <td>Ремонт компьютеров</td>\n",
       "      <td>видно имя?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>573</td>\n",
       "      <td>13</td>\n",
       "      <td>Ремонт компьютеров</td>\n",
       "      <td>рейтинг нулевой</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    contract  service_id            service_name  \\\n",
       "1        364          16  Сборка и ремонт мебели   \n",
       "2        370          13      Ремонт компьютеров   \n",
       "9        550          13      Ремонт компьютеров   \n",
       "11       568          13      Ремонт компьютеров   \n",
       "13       573          13      Ремонт компьютеров   \n",
       "\n",
       "                     user_request  \n",
       "1   заказ для Александра Трушкина  \n",
       "2               заказ для Алишера  \n",
       "9                       где фото?  \n",
       "11                     видно имя?  \n",
       "13                рейтинг нулевой  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data4 = data[~data.service_id.isin([14,10,4])].copy()\n",
    "data4.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.loc[~data.service_id.isin([14,10,4]), 'service_id'] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7f9fb10b70b8>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD+CAYAAAAzmNK6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAADp5JREFUeJzt3X+s3XV9x/HnSxCzTTOKXDts68pc51L+GJobIGF/aJhQMLGaTAZ/QEdY6h800YQsq+wPjIaEP6YmZoykjs6SOAiJGhttxMr8EbMgvRAGFGTc8GNtU9qrJWji4ga898f5dh4ut73n3Ht6Ti+f5yM5Od/z/n6+3/P+npzcV7+/TlNVSJLa85ZJNyBJmgwDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktSoMyfdwMmce+65tX79+km3IUkrysMPP/zzqppabNxpHQDr169nZmZm0m1I0oqS5IVBxnkISJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktSo0/pGsFNh/fbvTLqFgTx/+0cm3YKkNzn3ACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJatSiAZBkXZIfJHkyyf4kn+rqn01yKMmj3eOqvmU+k2Q2ydNJruirb+pqs0m2n5pNkiQNYpAbwV4Bbq6qR5K8A3g4yd5u3peq6h/6ByfZCFwDXAC8G/h+kj/pZt8BfBg4COxLsruqnhzFhkiShrNoAFTVYeBwN/2rJE8Ba06yyGbg3qr6DfBcklngom7ebFU9C5Dk3m6sASBJEzDUT0EkWQ+8H/gpcCmwLcn1wAy9vYSX6IXDg32LHeS3gXFgXv3iJXWt04Y/rSGtXAOfBE7yduDrwKer6pfAncB7gQvp7SF8YRQNJdmaZCbJzNzc3ChWKUlawEABkOSt9P74f62qvgFQVUeq6tWqeg34Cr89zHMIWNe3+NqudqL661TVjqqarqrpqampYbdHkjSgQa4CCnAX8FRVfbGvfl7fsI8DT3TTu4FrkrwtyfnABuAhYB+wIcn5Sc6id6J492g2Q5I0rEHOAVwKXAc8nuTRrnYLcG2SC4ECngc+CVBV+5PcR+/k7ivATVX1KkCSbcD9wBnAzqraP8JtkSQNYZCrgH4CZIFZe06yzG3AbQvU95xsOUnS+HgnsCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNWqon4OWdGr589oaJ/cAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhq1aAAkWZfkB0meTLI/yae6+jlJ9iZ5pnte1dWT5MtJZpM8luQDfeva0o1/JsmWU7dZkqTFDLIH8Apwc1VtBC4BbkqyEdgOPFBVG4AHutcAVwIbusdW4E7oBQZwK3AxcBFw6/HQkCSN36IBUFWHq+qRbvpXwFPAGmAzsKsbtgv4WDe9Gbi7eh4Ezk5yHnAFsLeqjlXVS8BeYNNIt0aSNLChzgEkWQ+8H/gpsLqqDnezXgRWd9NrgAN9ix3saieqz3+PrUlmkszMzc0N054kaQgDB0CStwNfBz5dVb/sn1dVBdQoGqqqHVU1XVXTU1NTo1ilJGkBAwVAkrfS++P/tar6Rlc+0h3aoXs+2tUPAev6Fl/b1U5UlyRNwCBXAQW4C3iqqr7YN2s3cPxKni3At/rq13dXA10CvNwdKrofuDzJqu7k7+VdTZI0AWcOMOZS4Drg8SSPdrVbgNuB+5LcCLwAXN3N2wNcBcwCvwZuAKiqY0k+D+zrxn2uqo6NZCskSUNbNACq6idATjD7sgXGF3DTCda1E9g5TIOSpFPDO4ElqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSoxYNgCQ7kxxN8kRf7bNJDiV5tHtc1TfvM0lmkzyd5Iq++qauNptk++g3RZI0jEH2AL4KbFqg/qWqurB77AFIshG4BrigW+afkpyR5AzgDuBKYCNwbTdWkjQhZy42oKp+nGT9gOvbDNxbVb8BnksyC1zUzZutqmcBktzbjX1y6I4lSSOxnHMA25I81h0iWtXV1gAH+sYc7Gonqr9Bkq1JZpLMzM3NLaM9SdLJLDUA7gTeC1wIHAa+MKqGqmpHVU1X1fTU1NSoVitJmmfRQ0ALqaojx6eTfAX4dvfyELCub+jarsZJ6pKkCVjSHkCS8/pefhw4foXQbuCaJG9Lcj6wAXgI2AdsSHJ+krPonSjevfS2JUnLtegeQJJ7gA8C5yY5CNwKfDDJhUABzwOfBKiq/Unuo3dy9xXgpqp6tVvPNuB+4AxgZ1XtH/nWSJIGNshVQNcuUL7rJONvA25boL4H2DNUd5KkU8Y7gSWpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjFg2AJDuTHE3yRF/tnCR7kzzTPa/q6kny5SSzSR5L8oG+ZbZ0459JsuXUbI4kaVCD7AF8Fdg0r7YdeKCqNgAPdK8BrgQ2dI+twJ3QCwzgVuBi4CLg1uOhIUmajEUDoKp+DBybV94M7OqmdwEf66vfXT0PAmcnOQ+4AthbVceq6iVgL28MFUnSGC31HMDqqjrcTb8IrO6m1wAH+sYd7Gonqr9Bkq1JZpLMzM3NLbE9SdJiln0SuKoKqBH0cnx9O6pquqqmp6amRrVaSdI8Sw2AI92hHbrno139ELCub9zarnaiuiRpQpYaALuB41fybAG+1Ve/vrsa6BLg5e5Q0f3A5UlWdSd/L+9qkqQJOXOxAUnuAT4InJvkIL2reW4H7ktyI/ACcHU3fA9wFTAL/Bq4AaCqjiX5PLCvG/e5qpp/YlmSNEaLBkBVXXuCWZctMLaAm06wnp3AzqG6kySdMt4JLEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMW/f8AJGklWr/9O5NuYSDP3/6Rib23ewCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVHLCoAkzyd5PMmjSWa62jlJ9iZ5pnte1dWT5MtJZpM8luQDo9gASdLSjGIP4ENVdWFVTXevtwMPVNUG4IHuNcCVwIbusRW4cwTvLUlaolNxCGgzsKub3gV8rK9+d/U8CJyd5LxT8P6SpAEsNwAK+F6Sh5Ns7Wqrq+pwN/0isLqbXgMc6Fv2YFd7nSRbk8wkmZmbm1tme5KkE1nufwn551V1KMm7gL1JftY/s6oqSQ2zwqraAewAmJ6eHmpZSdLglrUHUFWHuuejwDeBi4Ajxw/tdM9Hu+GHgHV9i6/tapKkCVhyACT5vSTvOD4NXA48AewGtnTDtgDf6qZ3A9d3VwNdArzcd6hIkjRmyzkEtBr4ZpLj6/nXqvpukn3AfUluBF4Aru7G7wGuAmaBXwM3LOO9JUnLtOQAqKpngT9boP4L4LIF6gXctNT3kySNlncCS1KjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEaNPQCSbErydJLZJNvH/f6SpJ6xBkCSM4A7gCuBjcC1STaOswdJUs+49wAuAmar6tmq+h/gXmDzmHuQJAGpqvG9WfKXwKaq+pvu9XXAxVW1rW/MVmBr9/J9wNNja3DpzgV+Pukm3kT8PEfLz3N0Vspn+YdVNbXYoDPH0ckwqmoHsGPSfQwjyUxVTU+6jzcLP8/R8vMcnTfbZznuQ0CHgHV9r9d2NUnSmI07APYBG5Kcn+Qs4Bpg95h7kCQx5kNAVfVKkm3A/cAZwM6q2j/OHk6RFXXIagXw8xwtP8/ReVN9lmM9CSxJOn14J7AkNcoAkKRGGQCS1KjT7j4AtS3J3VV1/aT7kACS/Cm9XytY05UOAbur6qnJdTU6BoAmJsn8S4ADfCjJ2QBV9dHxdyX1JPk74Fp6P1nzUFdeC9yT5N6qun1izY2IVwGNQJJ3VdXRSfex0iR5BHgS+Geg6AXAPfTuD6GqfjS57tS6JP8JXFBV/zuvfhawv6o2TKaz0fEcwJCSnDPv8U7goSSrkpwz6f5WmGngYeDvgZer6ofAf1fVj/zjvzRJfj/J7Ul+luRYkl8keaqrnT3p/laY14B3L1A/r5u34rkHMKQkrwEvzCuvBQ4CVVV/NP6uVrYka4EvAUeAj1bVeybc0oqV5H7g34BdVfViV/sDYAtwWVVdPsn+VpIkm4B/BJ4BDnTl9wB/DGyrqu9OqrdRMQCGlORm4MPA31bV413tuao6f7KdrXxJPgJcWlW3TLqXlSrJ01X1vmHnaWFJ3kLvZ+z7TwLvq6pXJ9fV6BgAS9D3L9YDwK3Af/gvf50OknwP+D69PYAjXW018NfAh6vqLybYnk4zngNYgqo6WFWfAH4I7AV+d7IdSf/vr4B3Aj/qzgEco/c9PQf4xCQb0+nHPYBlSvI7wHur6okkN1TVv0y6J2khfj81nwEwQkn+yxOYOl35/dR83gg2pCSPnWgWsHqcvUjz+f3UMAyA4a0GrgBemlcP8O/jb0d6Hb+fGpgBMLxvA2+vqkfnz0jyw/G3I72O308NzHMAktQoLwOVpEYZAJLUKANAkhplAEhSo/4PJkXAUrsSMJAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "data.service_id.value_counts().plot(kind='bar')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "vc = data4.service_id.value_counts().sort_values(ascending=False)[5:].index.tolist()\n",
    "ind = data4[data4['service_id'].isin(vc)].index\n",
    "data4.drop(ind, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7f9fb10382b0>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD/CAYAAAD/qh1PAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAES1JREFUeJzt3X+QXWV9x/H3RyIoWgXMEjEBEzVo0VahK6XDaFFajcIYprUO1GpqY9NWRFsdEawj7R90cFpL0bZ2IgTCDD9kqG0YQS0iythKcEHkp0qGX0kmkHXwRy0zaODbP+7B7CxJNrl3d2949v2ayew5z3nOPd85c/O5zz57zrmpKiRJ7XrGsAuQJM0sg16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekho3b9gFAMyfP78WL1487DIk6Wnl5ptv/mFVjUzVb68I+sWLFzM2NjbsMiTpaSXJA7vTz6kbSWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuOmvGEqyRrgRGBrVb1qQvtpwKnA48DVVXV6134msLJr/0BVfWUmCp9s8RlXz8Zhdun+c04YdgmS9BS7c2fsRcA/Axc/2ZDkDcBy4NVV9ViSg7v2I4CTgVcCLwK+muTwqnp8uguXJO2eKaduquoG4JFJzX8BnFNVj3V9tnbty4HLq+qxqroP2AAcPY31SpL2UL9z9IcDr0uyPsk3kry2a18IbJzQb1PXJkkakn4fajYPOAg4BngtcEWSl+zJCyRZBawCOOyww/osQ5I0lX5H9JuAL1TPTcATwHxgM3DohH6LuranqKrVVTVaVaMjI1M+ZVOS1Kd+g/4/gTcAJDkc2Bf4IXAVcHKS/ZIsAZYCN01HoZKk/uzO5ZWXAccB85NsAs4C1gBrktwB/BxYUVUF3JnkCuAuYBtwqlfcSNJwTRn0VXXKTjb90U76nw2cPUhRkqTp452xktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGTRn0SdYk2dp9m9TkbR9OUknmd+tJ8ukkG5LcluSomShakrT7dmdEfxGwbHJjkkOBNwEPTmh+C73viV0KrAI+O3iJkqRBTBn0VXUD8MgONp0LnA7UhLblwMXVcyNwQJJDpqVSSVJf+pqjT7Ic2FxV3520aSGwccL6pq5NkjQkU345+GRJ9gc+Rm/apm9JVtGb3uGwww4b5KUkSbvQz4j+pcAS4LtJ7gcWAbckeSGwGTh0Qt9FXdtTVNXqqhqtqtGRkZE+ypAk7Y49Dvqqur2qDq6qxVW1mN70zFFV9RBwFfDu7uqbY4CfVNWW6S1ZkrQndufyysuAbwEvT7IpycpddL8GuBfYAHwOeN+0VClJ6tuUc/RVdcoU2xdPWC7g1MHLkiRNF++MlaTGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMbtzlcJrkmyNckdE9r+Psn3ktyW5D+SHDBh25lJNiT5fpI3z1ThkqTdszsj+ouAZZPargVeVVW/DvwAOBMgyRHAycAru33+Nck+01atJGmPTRn0VXUD8Miktv+qqm3d6o3Aom55OXB5VT1WVffR+5Lwo6exXknSHpqOOfo/Ab7ULS8ENk7YtqlrkyQNyUBBn+SvgW3AJX3suyrJWJKx8fHxQcqQJO1C30Gf5I+BE4F3VlV1zZuBQyd0W9S1PUVVra6q0aoaHRkZ6bcMSdIU+gr6JMuA04G3VdWjEzZdBZycZL8kS4ClwE2DlylJ6te8qTokuQw4DpifZBNwFr2rbPYDrk0CcGNV/XlV3ZnkCuAuelM6p1bV4zNVvCRpalMGfVWdsoPmC3bR/2zg7EGKkiRNH++MlaTGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1bsqgT7ImydYkd0xoOyjJtUnu6X4e2LUnyaeTbEhyW5KjZrJ4SdLUdmdEfxGwbFLbGcB1VbUUuK5bB3gLve+JXQqsAj47PWVKkvo1ZdBX1Q3AI5OalwNru+W1wEkT2i+unhuBA5IcMl3FSpL2XL9z9Auqaku3/BCwoFteCGyc0G9T1yZJGpKB/xhbVQXUnu6XZFWSsSRj4+Pjg5YhSdqJfoP+4SenZLqfW7v2zcChE/ot6tqeoqpWV9VoVY2OjIz0WYYkaSr9Bv1VwIpueQWwbkL7u7urb44BfjJhikeSNATzpuqQ5DLgOGB+kk3AWcA5wBVJVgIPAO/oul8DvBXYADwKvGcGapYk7YEpg76qTtnJpuN30LeAUwctSpI0fbwzVpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDVuyssr9fSz+Iyrh10C959zwrBLkNRxRC9JjTPoJalxTt2oaU5jSY7oJal5Br0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklq3EBBn+SvktyZ5I4klyV5VpIlSdYn2ZDk80n2na5iJUl7ru+gT7IQ+AAwWlWvAvYBTgY+CZxbVS8DfgSsnI5CJUn9GXTqZh7w7CTzgP2BLcAbgSu77WuBkwY8hiRpAH0HfVVtBv4BeJBewP8EuBn4cVVt67ptAhbuaP8kq5KMJRkbHx/vtwxJ0hQGmbo5EFgOLAFeBDwHWLa7+1fV6qoararRkZGRfsuQJE1hkKmb3wHuq6rxqvoF8AXgWOCAbioHYBGwecAaJUkDGCToHwSOSbJ/kgDHA3cB1wNv7/qsANYNVqIkaRCDzNGvp/dH11uA27vXWg18FPhQkg3AC4ALpqFOSVKfBnoefVWdBZw1qfle4OhBXleSNH28M1aSGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGDfQ8eklPH4vPuHrYJXD/OScMu4Q5aaARfZIDklyZ5HtJ7k7yW0kOSnJtknu6nwdOV7GSpD036NTNecCXq+oVwKuBu4EzgOuqailwXbcuSRqSvoM+yfOB19N9J2xV/byqfgwsB9Z23dYCJw1apCSpf4OM6JcA48CFSb6T5PwkzwEWVNWWrs9DwIJBi5Qk9W+QoJ8HHAV8tqqOBP6PSdM0VVVA7WjnJKuSjCUZGx8fH6AMSdKuDHLVzSZgU1Wt79avpBf0Dyc5pKq2JDkE2LqjnatqNbAaYHR0dIcfBpI0E+baFUh9j+ir6iFgY5KXd03HA3cBVwErurYVwLqBKpQkDWTQ6+hPAy5Jsi9wL/Aeeh8eVyRZCTwAvGPAY0iSBjBQ0FfVrcDoDjYdP8jrSpKmj49AkKTGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1buCgT7JPku8k+WK3viTJ+iQbkny++/YpSdKQTMeI/oPA3RPWPwmcW1UvA34ErJyGY0iS+jRQ0CdZBJwAnN+tB3gjcGXXZS1w0iDHkCQNZtAR/T8BpwNPdOsvAH5cVdu69U3AwgGPIUkaQN9Bn+REYGtV3dzn/quSjCUZGx8f77cMSdIUBhnRHwu8Lcn9wOX0pmzOAw5IMq/rswjYvKOdq2p1VY1W1ejIyMgAZUiSdqXvoK+qM6tqUVUtBk4GvlZV7wSuB97edVsBrBu4SklS32biOvqPAh9KsoHenP0FM3AMSdJumjd1l6lV1deBr3fL9wJHT8frSpIG552xktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1Li+gz7JoUmuT3JXkjuTfLBrPyjJtUnu6X4eOH3lSpL21CAj+m3Ah6vqCOAY4NQkRwBnANdV1VLgum5dkjQkfQd9VW2pqlu65f8F7gYWAsuBtV23tcBJgxYpSerftMzRJ1kMHAmsBxZU1ZZu00PAguk4hiSpPwMHfZLnAv8O/GVV/XTitqoqoHay36okY0nGxsfHBy1DkrQTAwV9kmfSC/lLquoLXfPDSQ7pth8CbN3RvlW1uqpGq2p0ZGRkkDIkSbswyFU3AS4A7q6qf5yw6SpgRbe8AljXf3mSpEHNG2DfY4F3AbcnubVr+xhwDnBFkpXAA8A7BitRkjSIvoO+qr4JZCebj+/3dSVJ08s7YyWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS4wx6SWqcQS9JjZuxoE+yLMn3k2xIcsZMHUeStGszEvRJ9gH+BXgLcARwSpIjZuJYkqRdm6kR/dHAhqq6t6p+DlwOLJ+hY0mSdmGmgn4hsHHC+qauTZI0y1JV0/+iyduBZVX13m79XcBvVtX7J/RZBazqVl8OfH/aC9lz84EfDruIvYTnYjvPxXaei+32hnPx4qoamarTvBk6+Gbg0Anri7q2X6qq1cDqGTp+X5KMVdXosOvYG3gutvNcbOe52O7pdC5maurm28DSJEuS7AucDFw1Q8eSJO3CjIzoq2pbkvcDXwH2AdZU1Z0zcSxJ0q7N1NQNVXUNcM1Mvf4M2aumkobMc7Gd52I7z8V2T5tzMSN/jJUk7T18BIIkNc6gl6TGGfSS1LgZ+2Os1IIkB1fV1mHXMQxJXgL8Hr17Yh4HfgBcWlU/HWph2mNzckSfZNmE5ecnuSDJbUkuTbJgmLXtTZJ8adg1zKYkB0369wLgpiQHJjlo2PXNpiQfAP4NeBbwWmA/eoF/Y5Ljhlia+jAnr7pJcktVHdUtnw88BHyO3ujlt6vqpGHWN5uSHLWzTcAXq+qQ2axnmJI8ATwwqXkRvWc1VVW9ZParGo4ktwOvqarHk+wPXFNVxyU5DFhXVUcOucRZleSFwFnAE8AngNOA3wfuBj5YVVuGWN6UnLqB0ap6Tbd8bpIVQ61m9n0b+Aa9YJ/sgFmuZdg+Avwu8JGquh0gyX1VtWS4ZQ3NPHpTNvsBzwWoqgeTPHOoVQ3HRcDVwHOA64FLgLcCJ9H7zWevfjrvXA36g5N8iF64PS9JavuvNnNtOutu4M+q6p7JG5Js3EH/ZlXVp5J8nt4H/kZ6I7i59ytvz/nAt5OsB14HfBIgyQjwyDALG5IFVfUZgCTvq6pPdu2fSbJyiHXtlrka9J8DfqVbXkvvKXTj3a9ntw6tquH4G3b+4XbaLNaxV6iqTcAfJHkbcC2w/5BLGoqqOi/JV4FfBT5VVd/r2seB1w+1uOGY+H/k4knb9pnNQvoxJ4O+qv42ySvoPSN/fVX9rGt/KMmlw61u1m0EtgAkeTZwJnAkcBfwd0OsaygmvC++Ri/oX9q1L6uqLw+zttnWPZ/KZ1T1rEvy3Kr6WVV9/MnGJC9j73jE+i7NtWkKAJKcBqyjN2K9I8nE+bW5Fm5rgEe75fOA59H7Nf1R4MJhFTUM3ZUmv3xfAG+qqju6zXPtfaEJquoTTw4IJ7VvoDd3v1ebkyN6el948htV9bMki4ErkyyuqvPY8R8lW/aMqtrWLY8+eTUS8M0kc20a60/xfaE997fs5YOiuRr0z5gwXXN/d13wlUlezNz7D31HkvdU1YXAd5OMVtVYksOBXwy7uFnm+0I7lOS2nW0C9vp7b+Zq0D+c5DVVdStAN4I7kd40xq8Nt7RZ917gvCQfp/e1aN/qrjjZ2G2bS3xfaGcWAG8GfjSpPcD/zH45e2au3jC1CNhWVQ/tYNuxVfXfQyhrqJI8D1hC78N/U1U9POSSZp3vC+1MkguAC6vqmzvYdmlV/eEQytptczLoJWkumZNX3UjSXGLQS1LjDHpJapxBL0mNM+glqXH/Dzbng1CiNXoOAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "data4.service_id.value_counts().plot(kind='bar')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "REPLACE_BY_SPACE_RE = re.compile('[/(){}\\[\\]\\|@,;]')\n",
    "BAD_SYMBOLS_RE = re.compile('[^а-я ]')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_stopwords(text: str) -> str:\n",
    "        try:\n",
    "            text = text.lower()\n",
    "            text = REPLACE_BY_SPACE_RE.sub(' ', text)\n",
    "            text = BAD_SYMBOLS_RE.sub('', text)\n",
    "            return \" \".join([token for token in text.split() if token not in mystopwords])\n",
    "        except:\n",
    "            return \"\"\n",
    "        \n",
    "def lemmatize(text: str) -> str:\n",
    "    try:\n",
    "        return \"\".join(m.lemmatize(text)).strip()  \n",
    "    except:\n",
    "        return \" \"\n",
    "\n",
    "def remove_stoplemmas(text: str) -> str:\n",
    "    try:\n",
    "        return \" \".join([token for token in text.split() if token not in mystoplemmas])\n",
    "    except:\n",
    "        return \"\"\n",
    "\n",
    "def clear_text(text: str) ->str:\n",
    "    cText = remove_stopwords(text)\n",
    "    cText = lemmatize(cText)\n",
    "    cText = remove_stoplemmas(cText)\n",
    "    return cText"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "m = Mystem()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to /home/mic/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import nltk\n",
    "nltk.download('stopwords')\n",
    "mystopwords = stopwords.words('russian') + ['это', 'наш', 'тыс', 'млн', 'млрд', 'также',  'т', 'д','iKomek.kz','кв']\n",
    "mystoplemmas = ['который', 'прошлый', 'сей', 'свой', 'наш', 'мочь']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['lemmas'] = data.user_request.apply(lambda x: clear_text(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>contract</th>\n",
       "      <th>service_id</th>\n",
       "      <th>service_name</th>\n",
       "      <th>user_request</th>\n",
       "      <th>lemmas</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>354</td>\n",
       "      <td>4</td>\n",
       "      <td>Сантехник</td>\n",
       "      <td>лалсо</td>\n",
       "      <td>лалсо</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>364</td>\n",
       "      <td>0</td>\n",
       "      <td>Сборка и ремонт мебели</td>\n",
       "      <td>заказ для Александра Трушкина</td>\n",
       "      <td>заказ александр трушкин</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>370</td>\n",
       "      <td>0</td>\n",
       "      <td>Ремонт компьютеров</td>\n",
       "      <td>заказ для Алишера</td>\n",
       "      <td>заказ алишер</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>379</td>\n",
       "      <td>14</td>\n",
       "      <td>Муж на час</td>\n",
       "      <td>для Ерлана</td>\n",
       "      <td>ерлан</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>385</td>\n",
       "      <td>14</td>\n",
       "      <td>Муж на час</td>\n",
       "      <td>мастер мастеров</td>\n",
       "      <td>мастер мастер</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   contract  service_id            service_name  \\\n",
       "0       354           4               Сантехник   \n",
       "1       364           0  Сборка и ремонт мебели   \n",
       "2       370           0      Ремонт компьютеров   \n",
       "3       379          14              Муж на час   \n",
       "4       385          14              Муж на час   \n",
       "\n",
       "                    user_request                   lemmas  \n",
       "0                          лалсо                    лалсо  \n",
       "1  заказ для Александра Трушкина  заказ александр трушкин  \n",
       "2              заказ для Алишера             заказ алишер  \n",
       "3                     для Ерлана                    ерлан  \n",
       "4                мастер мастеров            мастер мастер  "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "vec_model = TfidfVectorizer(analyzer='char', ngram_range={4,6})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = data.lemmas\n",
    "Y = data.service_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size=0.33, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_result(model, text, x_test, y_true, cv, proba=True):\n",
    "    print('Predicted service ID: ', model.predict([text]))\n",
    "    if proba:\n",
    "        pred_service = model.predict_proba([text])\n",
    "        print('Predic_proba : ', pred_service)\n",
    "    predictions = model.predict(x_test)\n",
    "    print(\"Precision: {0:6.2f}\".format(precision_score(y_true, predictions, average='macro')))\n",
    "    print(\"Recall: {0:6.2f}\".format(recall_score(y_true, predictions, average='macro')))\n",
    "    print(\"F1-measure: {0:6.2f}\".format(f1_score(y_true, predictions, average='macro')))\n",
    "    print(\"Accuracy: {0:6.2f}\".format(accuracy_score(y_true, predictions)))\n",
    "    print(classification_report(y_true, predictions))\n",
    "    labels = ['Другие','Сантехник','Электрик','Муж на час']\n",
    "    sns.heatmap(data=confusion_matrix(y_true, predictions), annot=True, fmt=\"d\", cbar=False, xticklabels=labels, yticklabels=labels)\n",
    "    plt.title(\"Confusion matrix\")\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "param = [\n",
    "         {\"C\":[5,10,50], 'penalty': ['l1'], 'solver': ['liblinear', 'saga']},\n",
    "         {\"C\":[5,10,50],'penalty': ['l2'], 'solver': ['newton-cg'] }\n",
    "        ]\n",
    "model = LogisticRegression(multi_class = 'auto', max_iter = 200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tuned hpyerparameters :(best parameters)  {'C': 5, 'penalty': 'l2', 'solver': 'newton-cg'}\n",
      "accuracy : 0.77535044602221\n"
     ]
    }
   ],
   "source": [
    "cv = GridSearchCV(model, param, cv = 10)\n",
    "logreg = Pipeline([\n",
    "                    ('tfidf', vec_model),\n",
    "                    ('clf', cv),\n",
    "               ])\n",
    "logreg.fit(X_train, y_train)\n",
    "print(\"tuned hpyerparameters :(best parameters) \", cv.best_params_)\n",
    "print(\"accuracy :\", cv.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test score:  0.7719881744271988\n"
     ]
    }
   ],
   "source": [
    "test_score = logreg.score(X_test, y_test)\n",
    "print('Test score: ', test_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0])"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_text = 'генеральная уборка 4-х комнатной квартиры'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted service ID:  [0]\n",
      "Predic_proba :  [[0.52766356 0.15625559 0.10216178 0.21391908]]\n",
      "Precision:   0.75\n",
      "Recall:   0.59\n",
      "F1-measure:   0.61\n",
      "Accuracy:   0.66\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.97      0.23      0.37       251\n",
      "           4       0.74      0.72      0.73       828\n",
      "          10       0.75      0.76      0.75       735\n",
      "          14       0.52      0.65      0.58       892\n",
      "\n",
      "   micro avg       0.66      0.66      0.66      2706\n",
      "   macro avg       0.75      0.59      0.61      2706\n",
      "weighted avg       0.69      0.66      0.65      2706\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXEAAAEICAYAAACpqsStAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3XmcTfXjx/HX584Yy4xlzDAYe1q+lBZLIpSyJupX6Ev795ukRaFNSiIVWZI1KaJvthZL+lJfkUoxCtlFiGmMdZgxlpn5/P64t2lUZgzDuZ/xfj4e9+Gezzn3nvc9zbzvuZ97yFhrERERN/m8DiAiIqdPJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuAQVY0xhY8xsY0ySMWb6GTxPJ2PM/LzM5hVjTENjzAavc0hwMrpOXE6HMaYj0B24BDgErABettZ+fYbPexfwKFDfWpt2xkGDnDHGAhdaa3/2Oou4SWfikmvGmO7AMGAAEANUBEYBbfPg6SsBG8+HAj8VxphQrzNIkLPW6qbbKd+A4kAy0C6bbQriL/n4wG0YUDCw7jpgB9ADSAR+A+4LrOsLHAOOB/bxL+BFYHKW564MWCA0sHwvsAX/p4FfgE5Zxr/O8rj6wDIgKfBn/SzrFgL9gG8CzzMfiD7Ja/s9/1NZ8t8CtAI2AvuAXlm2rwssAQ4Eth0BhAXWfRV4LSmB19shy/M/DSQAk34fCzzmgsA+rgoslwN2A9d5/bOhmzc3nYlLbl0DFAI+zmab54B6wBXA5fiLrHeW9WXwvxnE4i/qkcaYSGttH/xn91OttRHW2vHZBTHGhAPDgZbW2qL4i3rF32xXEvg0sG0UMAT41BgTlWWzjsB9QGkgDOiZza7L4D8GscALwDjgTqAW0BB43hhTJbBtOvAEEI3/2N0AdAWw1jYKbHN54PVOzfL8JfF/KumcdcfW2s34C36yMaYI8C4w0Vq7MJu8ko+pxCW3ooA9Nvvpjk7AS9baRGvtbvxn2HdlWX88sP64tXYu/rPQi08zTwZwqTGmsLX2N2vtmr/Z5iZgk7V2krU2zVr7AbAeuDnLNu9aazdaa1OBafjfgE7mOP75/+PAFPwF/Ya19lBg/2vxv3lhrV1urf0usN+twFig8Sm8pj7W2qOBPCew1o4Dfga+B8rif9OU85RKXHJrLxCdw1xtOWBbluVtgbHM5/jTm8BhICK3Qay1KfinILoAvxljPjXGXHIKeX7PFJtlOSEXefZaa9MD938v2V1Z1qf+/nhjzEXGmDnGmARjzEH8nzSis3lugN3W2iM5bDMOuBR401p7NIdtJR9TiUtuLQGO4p8HPpl4/FMBv6sYGDsdKUCRLMtlsq601s6z1jbFf0a6Hn+55ZTn90w7TzNTbozGn+tCa20xoBdgcnhMtpeMGWMi8H/PMB54MTBdJOcplbjkirU2Cf888EhjzC3GmCLGmALGmJbGmIGBzT4AehtjShljogPbTz7NXa4AGhljKhpjigPP/r7CGBNjjGkbmBs/in9aJuNvnmMucJExpqMxJtQY0wGoDsw5zUy5URQ4CCQHPiU89Kf1u4CquXzON4A4a+2/8c/1jznjlOIslbjkmrV2MP5rxHvjvzLiV+AR4JPAJv2BOGAV8BPwQ2DsdPb1OTA18FzLObF4fYEc8fiv2GjMX0sSa+1eoDX+K2L24r+ypLW1ds/pZMqlnvi/ND2E/1PC1D+tfxGYaIw5YIxpn9OTGWPaAi3443V2B64yxnTKs8TiFP1lHxERh+lMXETEYSpxERGHqcRFRBymEhcRcdhZ/8d1LipVW9+c5pGdKefiYorzR6gvxOsI+Ur76Ku8jpCvjNs6Pae/TwDoTFxExGkqcRERh6nERUQcphIXEXGYSlxExGEqcRERh6nERUQcphIXEXGYSlxExGEqcRERh6nERUQcphIXEXGYSlxExGEqcRERh6nERUQcphIXEXGYSlxExGEqcRERh6nERUQcphIXEXGYSlxExGEqcRERh4V6HcArC5bPIiX5MBkZ6aSlpXNb07v5x6UX0XfQsxQsFEZaWjp9n3qNVT+u8TpqUBs9ZiAtWzRh9+691KnTPHO8S5d76Pzg3aSnpzPvvwvo3ftVD1O6Iza2LGPGvU7p0lFYa5nw7lTGjJpAv/7P0KJVE44dO84vv2zn4S5PkZR0yOu4QemegQ9Rs0ktDu1N4sXmPQC4+fF2NLzjRpL3HQTgo4H/YfXCH4kqX4qXvhjGri3xAGz5cSOTnxvnWfbTcd6WOMDdtz7I/n1JmctPvvAYI14fx1f/+5bGNzbgyT6PcdctD3qYMPhNnjSDsWMmMm7ckMyxRo2uoXXrptS7uiXHjh2jVKkoDxO6JS0tjd7PDmDlyjVERISzaPFMvlzwNV8u+JoX+wwiPT2dvi89RfceD9HnhYFexw1K385YyJcT/8v9Qx45YfyL8XOYP272X7bfvS2Bl1o9ea7i5TlNp2RhsUQUDQcgomgEiQm7PU4U/L75Zin7srwRAvz7gU4MHjyaY8eOAbB7914vojlp167drFzp//SXnJzChg0/U65sDAsWfE16ejoAy5atoFxsGS9jBrVNS9eRkpTsdYxzJsczcWPMJUBbIDYwtBOYZa1ddzaDnW3WWt6ZPhJrLVMnfsTUSR8z4LnBjJ82gqdf7IbP56NDq/u9jumkCy+sSv0Gdenz4pMcOXKUXr1e5oflq7yO5ZyKFWOpeXkN4uJWnjB+512389GHn3qUyl3X39OCa/6vMVt/2sz0/u9x+GAKANEVSvP8pwNJTU5l5usfsGnZeo+T5k62JW6MeRr4JzAFWBoYLg98YIyZYq11dqKzY+t/sythNyWjI5kwfSSbf95Ki5tvYMDzQ5g/ZwEt297IgGHPc+/tD3sd1TmhISFERhbnusa3UKv25UyaNJIa1Rt6Hcsp4eFFmPT+KJ59uh+HDv1xVtnzya6kpaczbepMD9O5Z+Hk+cwZ/iFYS9sed9Cu991MfGo0SYn7ebr+Q6QcSKbipVV5+K0n6dOsO0eSU72OfMpymk75F1DHWvuqtXZy4PYqUDew7m8ZYzobY+KMMXFJR4JzSmJXYKpk3579fD53ITWvrMGtHVozf84CAD6b+QU1r6rhZURn7YxPYNbMeQAsj1tJRkYG0dElPU7ljtDQUCa9P5JpU2cye9b8zPGOnW6jeYvreeD+JzxM56ZDe5KwGRlYa1k85QuqXF4NgLRjaaQc8L9Jbl+9hd3bdxFTpayXUXMtpxLPAMr9zXjZwLq/Za19y1pb21pbu3ihUmeS76woXKQQ4eFFMu83uO5qNq3fTGLCburWrwXANQ3rsHXLr17GdNbs2fNp1LgeANWqVSEsrAB79uzzOJU7Rox6lQ0bNjNyxDuZYzfc2IhuTzzAHR0eJDX1iIfp3FS8VInM+1c2r8vOjf7f7YiSxTA+fw1GVyhN6cpl2b090ZOMpyunOfHHgf8ZYzYBvzdaRaAa8MhJHxXkoktFMXLCIABCQkOY/dE8Fi9YQu+U/jz3ck9CQ0I4evQYz3d/2eOkwW/ChOE0bFSPqKhINm5aQv/+Q3lv4jTGjBnIsmXzOHb8OJ0f6OF1TGfUu6YW/+x4K6tXr2fxt/4rKV56cTADB71AWMEwPpk1EYC4ZSt4otvzXkYNWg8M78ZF9WoQEVmUgUvGMGvoNC6qV4MK1SuDtezZsZvJvcYCcFHdf9C2ewfS09LJyMhg8nNvcdixL0WNtTb7DYzx4Z8+yfrF5jJrbfqp7OCiUrWz34Gcsp0pe7yOkK+E+kK8jpCvtI++yusI+cq4rdPNqWyX49Up1toM4LszTiQiInlO14mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDgsx//b/ZnakvTb2d7FeSM1frHXEfKVmCrNvY6Qr1ycUdDrCOclnYmLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw0K9DhAMmje7jiFDXiLE5+Oddz9g4KCRXkcKes1uu4fwIkXw+XyEhIQw7Z3hrN+0hX6D3uRw6hHKlS3Na32eIiI8nJ/WbuDF14YDYLF0vb8TNzZu4PErCE6xsWUY9dYgSpeOxlrLxHenMnb0RMZPGEa1C6sCULx4UZKSDtG4QRuP0wafomVL0nJoF8JLFcday6r/fMkP78yjUPFwWo96hOLlS5G0Yzezu77J0aTDhBUtzE1vPETRclH4QkOIGzuX1dO/8vpl5Iqx1p7VHYSGxZ7dHZwhn8/HujWLadHqn+zY8RvfLZnLnXd1Zd26TV5H+4vU+MVeR8jU7LZ7mDp+OJElimeOdfjXY/R85N/UubImH82Zx874XTza+W5SjxyhQGgBQkND2L1nH7fd05UFM98nNDTEw1cAMVWae7r/vxMTU4qYMqVYtXItERHhLFj8MXfd0ZUNG37O3KbfgGc4mJTMoNdGeJj0r3pFXu11BMJLlyC8dAkSV2+lQHgh7vq0HzMfGEqNdo04ciCFpaNmU7frzRQqXoSvXpnK1Q+3oWCxwnz1ylQKlyzK/QsHMbrWw2QcT/f6pdBz+2RzKtvlejrFGPNe7uMEr7p1rmTz5q388st2jh8/zrRpM2lzc/D9crtg2687qX3FZQBcU+cqPl/0NQCFCxXKLOyjx46BOaWfzfPSrl27WbVyLQDJySls3LCZsuViTtjmlltb8eGM2V7EC3opiQdIXL0VgOMpR9j3czwRZUpSrWkt1szwnwStmbGYas1qA/5PhmHhhQEICy/EkQMpZKRleJL9dGU7nWKMmfXnIeB6Y0wJAGut85/nysWW4dcd8ZnLO3b+Rt06V3qYyA3GGDo/8RzGGNq1bUm7tq24oEolFixewg2N6jP/y8Uk7NqTuf2qNet5fsBQ4ncl8srzPT0/C3dBhYqx1KxZneVxKzPHrmlQh8TEPWzZvM3DZG4oVj6a0jUq8duPmykSXYyUxAOAv+iLRBcD4McJn3Pr+O50iRtBWHgh5jw8As7y7ERey2lOvDywFngbsPhLvDYwOLsHGWM6A50BTEhxfL7wM08qQeW90a8TUyqavfsP8MDjvahSqQL9ej3BK0NHM3bCB1x3bT0KFPjjx6tmjUuY+f5YNm/dznP9B9OwXh0KFgzz8BUEt/DwIkycPIJez7zMoUPJmeO33d6aj2bM8TCZGwoUKUibsd34su9kjiWnnnS7Ko0vI3HtNqbdMYASlWJo9/7T7Fi6IdvHBJucplNqA8uB54Aka+1CINVau8hau+hkD7LWvmWtrW2trR3sBR6/M4EK5ctlLpePLUt8fIKHidwQUyoagKjIEtzQqD4/rd1A1UoVGDdsANPeeZNWNzamQmzZvzzugsoVKVK4MJu2bD3Hid0RGhrKxMkjmDFtFnNmzc8cDwkJoXWbZnz84VwP0wU/X2gIbcZ2Y93H37Lpv3EAHN5zkPDSJQD/vPnhPQcBuLRd48xtDmzbRdKvuyl5wV9/boNZtiVurc2w1g4F7gOeM8aMIJ9d0bIsbgXVqlWhcuUKFChQgPbt2zJ7zvycH3geO5x6hJSUw5n3v136AxdWrcze/f6PqxkZGYydOIX2t7QCYEd8Amlp/i+K4hN28cu2X4ktG/P3Ty4MHzmAjRs2M2rEuyeMX3d9fTZt3KKTjBw0H/Rv9v0cz/K3P8sc2/z5D9S4vSEANW5vyM+fLwfgYPweKjWoAUCR6GJEXlCWpO2J5z70GTilQrbW7gDaGWNuAg6e3UjnVnp6Ot0e783cT/9DiM/HhIlTWbt2o9exgtreffvp1qsfAOlp6bRqdh3X1qvNpGmfMOUj/0f9GxvX59abmgHww6o1jJ80jdDQUHw+Q++eD59wVYv84epranFHx1tZs3o9i77xfyXVr+9gvpi/iFtvb82H0zWVkp3YOhdR47aG7F63nbs/exmAxQOn8f2o2dw8+lEu69CYgzv3MPuhNwFYMvwTWg5+kHvmv4Ix8NUrU0ndn5zdLoLOeX+JoUuC6RLD/CAYLzF0WTBcYpifnLVLDEVEJHioxEVEHKYSFxFxmEpcRMRhKnEREYepxEVEHKYSFxFxmEpcRMRhKnEREYepxEVEHKYSFxFxmEpcRMRhKnEREYepxEVEHKYSFxFxmEpcRMRhKnEREYepxEVEHKYSFxFxmEpcRMRhKnEREYepxEVEHBbqdQA5dSUqNvE6Qr6yb9l4ryPkKz1uesvrCOclnYmLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4rBQrwMEg+bNrmPIkJcI8fl4590PGDhopNeRnDF6zEBatmjC7t17qVOnOQC9nnuc++67gz179gHwYp+BzJu30MOUwa/Fg70pUrgQIT4fISE+pgx6hlFT5vDRF98QWawoAI91akPDWpcCsHHrDl4a8wEpqUcwxvDBwKcpGFbAy5cQNDoN7MKlTa7i0N6DDGjeM3O88T0taHh3M2x6BqsX/MjMV9/PXBdZLorenw9h7rDp/G/cHC9in7bzvsR9Ph/D33iZFq3+yY4dv/HdkrnMnjOfdes2eR3NCZMnzWDsmImMGzfkhPERb47njTfGeZTKTeNfepzIYhEnjN3Zugn33tL0hLG09HSefWMCAx67l4urlOfAoWRCQ0LOZdSg9t2MRSyaOI+7hzycOXbhNTW4rGltXm35FGnH0oiIKnbCY/6v992sWbjiXEfNE+f9dErdOleyefNWfvllO8ePH2fatJm0ubm517Gc8c03S9m3L8nrGOeVJSvWcVGlWC6uUh6AEkUjCAk573+VM21euo7DScknjDXs1JTPR88k7VgaAMl7D2auq9msNnt/TSRh06/nNGdeOe//y5eLLcOvO+Izl3fs/I1y5cp4mCh/eLDLPXz//WeMHjOQEiWK5fyA850xPNj3TTr0fIUZ87/OHJ7y2SJue6I/L4yYxMHkwwBsjU/EGEOXl96kfY9XeOfj+V6ldkbpqmW5oO4l9PykP92m9qFizQsACCtSkKZd2jL3jRkeJzx9Z6XEjTGdjTFxxpi4jIyUs7ELCWJvj5vMpTUaUa9eKxISEnnl1d5eRwp6E1/uwbTBzzKq9yNM+WwRcWs20aFFIz4d9RLTB/ciOrIYr0/4EID09HR+WLeZVx6/j4kDerDg+5V8t2q9x68guPlCQggvHsHrt/TmkwGTuX/k4wDc9Hg7Foz/lGOHj3qc8PRlW+LGmOLGmFeNMeuNMfuMMXuNMesCYyVO9jhr7VvW2trW2to+X3jep85D8TsTqFC+XOZy+diyxMcneJjIfYmJe8jIyMBay7vvTKF2rcu9jhT0YqL8v05RJYrS5OrLWb1pK1ElihES4sPn83Fb02v5adNW/7bRkdSqXo3IYhEULhhGw6tqsG6Lm1MB58qBhL2smLcUgG0rN2MzMogoWZRKV1Tjlmc70ffrN7nu/lY0e/hWGt3t1nRqTmfi04D9wHXW2pLW2ijg+sDYtLMd7lxYFreCatWqULlyBQoUKED79m2ZPUcfT89EmTKlMu+3adOcNWs3epgm+B0+cpSU1COZ95esXEe1iuXYneW7hgXfr+DCiv6TjQZXVGfTtnhSjx4jLT2duLWbuKC8pgCzs2r+Mi6qVx2A0lXKEloglOR9hxjW/kX6XPsofa59lIXvzGX+yI/56r15HqfNnZyuTqlsrX0t64C1NgF4zRhz/9mLde6kp6fT7fHezP30P4T4fEyYOJW1Kp1TNmHCcBo2qkdUVCQbNy2hf/+hNGpYj5o1q2OtZdv2HTz2aC+vYwa1fQcO8fhrYwFIz8igZcPaXHtVDXq9MYH1v+zAGChXKooXunQEoFhEEe5u04SOT/l/NRvWqkGj2pd5lj/Y3Dv8MS6sV52IyKL0WzKKuUOns2Tal3Qa+BC95r1O+vE0JvUY5XXMPGOstSdfacx84AtgorV2V2AsBrgXaGqtvTGnHYSGxZ58B5IrBUN1HXBe2rdsvNcR8pUeN73ldYR8ZcTWqeZUtstpOqUDEAUsCsyJ7wMWAiWBdmeUUEREzli20ynW2v3A04HbCYwx9wHvnqVcIiJyCs7kEsO+eZZCREROS7Zn4saYVSdbBcTkfRwREcmNnK5OiQGa47+kMCsDfHtWEomIyCnLqcTnABHW2r/8yzDGmIVnJZGIiJyynL7Y/Fc26zrmfRwREcmN8/4fwBIRcZlKXETEYSpxERGHqcRFRBymEhcRcZhKXETEYSpxERGHqcRFRBymEhcRcZhKXETEYSpxERGHqcRFRBymEhcRcZhKXETEYSpxERGHGWvtWd1BaFjs2d3BeaR7uUZeR8hXttlUryPkK5O+1v92Ny+FVbjcnMp2OhMXEXGYSlxExGEqcRERh6nERUQcphIXEXGYSlxExGEqcRERh6nERUQcphIXEXGYSlxExGEqcRERh6nERUQcphIXEXGYSlxExGEqcRERh6nERUQcphIXEXGYSlxExGEqcRERh6nERUQcphIXEXGYSlxExGGhXgfwWvny5ZjwzhuUjonGWsvbb7/PmyPGex0rqLUb+CD/aHIlyXsPMqT5UwB0GvEYpaqWBaBQsXCOHExhWKtn8YWGcPtrnYmtURlfaAg/fLSYL0fN9DJ+0Oky6BGualKbg3uT6NmsGwDte3SkdtO62AxL0t4kRvd4g/2J+6ndtC7te3TEZljS09OZ2Hc8G+LWefwKgkvzTg9TpHAhQkJ8hISEMHXUq/TsN5StO+IBOJR8mKIRRZgxdhDHj6fRd9hbrNmwGZ/PxzNd76XOFTU8fgW5c96XeFpaGk8+1ZcfV6wmIiKcpd//ly/+9xXr1m3yOlrQipuxiG8nzqPDkK6ZY+8/Mjzzfuvn7uTIocMA1Gx1NaFhoQxt8TQFCoXR44vXWTHrG/bv2HPOcwerRdMXMG/iXB4e0i1zbPbYj5k2+D8AtLj3Jm7r1oG3nxvDT9+sIu7zpQBUvKQSj498ku43POJJ7mD2zuA+RBYvlrn8+vNPZN4fNOY9IsKLADBj7hcAfPz2YPbuT+KhXgOYMvIVfD53JincSXqWJCQk8uOK1QAkJ6ewfv0mYsuV8ThVcPtl6XoOJyWfdH3Nm+qxYta3mcthhQviC/FRoFAY6cfSOHIo9VzEdMa6pWtJPnDi8UxN/uMYFSpSCGstAEcPH8kcL1ikEGDPScb8wlrLvEVLaHV9AwA2b9vB1VdcCkBUZHGKRYSzZuMWLyPm2nl/Jp5VpUrlueLyS/l+6Y9eR3FWlbqXkLwniT1bEwBYNfd7qjetRe+lowkrHMbsfpNITUrxOKUbOjzZiUb/dz2ph1Loe8fzmeN1ml/NP5+6i+LRxXn1vv4eJgxOxsCDT78MBtrd1JR2rW/MXLf8p3VERRanUnn/1N/FVSvz5ZI4WjZpQELiXtZu3EJC4h4uu6SaV/FzLdclbowpba1NPBthvBQeXoRpU8fRvWcfDh06+VmmZO+KNvVPOAuvcPkF2PQM+l/dlcLFw+k6rQ+bvl7Nvl/z3Y9Qnps66H2mDnqfW7reRot7WjF96BQAls37nmXzvucfdavToUdH+nfq43HS4DJxWD9iokuyd38SnZ/uT5WK5ahdszoAny34JvMsHODWltezZfsO7uj6DGVLl+LyGhfjC3FrgiLbtMaYkn+6RQFLjTGRxpiS2TyuszEmzhgTl5ER/GddoaGhTJ86jg8++JhPPvnM6zjO8oX4uLR5XVbOWZI5dmXbBmxYtJKMtHRS9h5k6/KNlK9Z1cOU7ln8ySKubnnNX8bXLV1L6YoxFI0s6kGq4BUT7a+mqMji3NCgDqvX/wxAWno6X3y9lObX1c/cNjQkhKe73suMsYN4s99THEpOoXL5cp7kPl05veXsAZZnucUBscAPgft/y1r7lrW2trW2ts8XnldZz5pxbw1m3fqfGfbGW15HcVq1ay9j95Z4khL2ZY4diN/DBfX93/YXKFyQildWI3FzvFcRnVGmctnM+3WaXc3OzTsBiKn0x/c1VS6tSoGwAhzaf+ic5wtWh1OPkHI4NfP+t8tXUa1yRQC+W/4TVSqWo0ypqMztU48c5XCq/3uGb5evIiQkhAsqlT/3wc9ATtMpTwJNgSettT8BGGN+sdZWOevJzpEG9etw1523s+qntcQtmw/A88+/ymf/XeBxsuDVcfijVK33D8Iji9JryQg+HzqDZdMWcsXN15wwlQLw7XvzaT+oC93nD8IYiJu+iIT12z1KHpweG96d6tdcStHIYoz67m2mD53CldfXolzVcmRkWPbs3M24XqMBuLrlNTS67XrSj6dz7OhRhj38usfpg8ve/Uk8/qL/mKSnp9OqybVcW/cKAD5beOJUCsC+A0l0eeZljM9H6aiSvPKMe1f6mN+/9T7pBsaUB4YCvwJ9gJXW2lP+PBwaFquvz/NI93KNvI6Qr2yzukomL036uq/XEfKVsAqXm1PZLscZfGvtDmttO2Ah8DlQ5MyiiYhIXjnlr2GttbOA64EbAYwx952tUCIicmpydS2NtTbVWrs6sKjPTiIiHsv2i01jzKqTrQJi8j6OiIjkRk5Xp8QAzYH9fxo3wLd/3VxERM6lnEp8DhBhrV3x5xXGmIVnJZGIiJyybEvcWvuvbNZ1zPs4IiKSG279IwEiInIClbiIiMNU4iIiDlOJi4g4TCUuIuIwlbiIiMNU4iIiDlOJi4g4TCUuIuIwlbiIiMNU4iIiDlOJi4g4TCUuIuIwlbiIiMNU4iIiDlOJi4g4zFhrvc4QFIwxna21b3mdI7/Q8cw7OpZ5K78dT52J/6Gz1wHyGR3PvKMOjgMbAAACOElEQVRjmbfy1fFUiYuIOEwlLiLiMJX4H/LNHFmQ0PHMOzqWeStfHU99sSki4jCdiYuIOEwlLiLiMJU4YIxpYYzZYIz52RjzjNd5XGeMCTHG/GiMmeN1FtcYY94xxiQaY1ZnGStpjPncGLMp8Geklxld8nfHM8u6HsYYa4yJ9iJbXjnvS9wYEwKMBFoC1YF/GmOqe5vKed2AdV6HcNQEoMWfxp4B/metvRD4X2BZTs0E/no8McZUAJoB2891oLx23pc4UBf42Vq7xVp7DJgCtPU4k7OMMeWBm4C3vc7iImvtV8C+Pw23BSYG7k8EbjmnoRx2kuMJMBR4CnD+yg6VOMQCv2ZZ3hEYk9MzDP8vR4bXQfKRGGvtb4H7CUCMl2FcZ4xpC+y01q70OkteUIlLnjHGtAYSrbXLvc6SX1n/NcHOnz16xRhTBOgFvOB1lryiEoedQIUsy+UDY5J7DYA2xpit+KelmhhjJnsbKV/YZYwpCxD4M9HjPC67AKgCrAz8nJYHfjDGlPE01RlQicMy4EJjTBVjTBhwBzDL40xOstY+a60tb62tjP84LrDW3ulxrPxgFnBP4P49wEwPszjNWvuTtba0tbZy4Od0B3CVtTbB42in7bwvcWttGvAIMA//FRXTrLVrvE0l5ytjzAfAEuBiY8wOY8y/gFeBpsaYTcCNgWU5BSc5nvmK/tq9iIjDzvszcRERl6nERUQcphIXEXGYSlxExGEqcRERh6nERUQcphIXEXHY/wNyP9u5CfZxtQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "print_result(logreg, test_text, X_test, y_test, cv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tuned hpyerparameters :(best parameters)  {'max_depth': 20, 'n_estimators': 50}\n",
      "accuracy : 0.6602949208083014\n"
     ]
    }
   ],
   "source": [
    "param = {\"n_estimators\":[5,10,50], 'max_depth': [5,10,20]}      \n",
    "model = RandomForestClassifier(min_samples_leaf=20, max_features=0.5)\n",
    "cv = GridSearchCV(model, param, cv = 3, n_jobs = 7)\n",
    "rf = Pipeline([\n",
    "                    ('tfidf', vec_model),\n",
    "                    ('clf', cv),\n",
    "               ])\n",
    "rf.fit(X_train, y_train)\n",
    "print(\"tuned hpyerparameters :(best parameters) \", cv.best_params_)\n",
    "print(\"accuracy :\", cv.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted service ID:  [0]\n",
      "Predic_proba :  [[0.52766356 0.15625559 0.10216178 0.21391908]]\n",
      "Precision:   0.75\n",
      "Recall:   0.59\n",
      "F1-measure:   0.61\n",
      "Accuracy:   0.66\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.97      0.23      0.37       251\n",
      "           4       0.74      0.72      0.73       828\n",
      "          10       0.75      0.76      0.75       735\n",
      "          14       0.52      0.65      0.58       892\n",
      "\n",
      "   micro avg       0.66      0.66      0.66      2706\n",
      "   macro avg       0.75      0.59      0.61      2706\n",
      "weighted avg       0.69      0.66      0.65      2706\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXEAAAEICAYAAACpqsStAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3XmcTfXjx/HX584Yy4xlzDAYe1q+lBZLIpSyJupX6Ev795ukRaFNSiIVWZI1KaJvthZL+lJfkUoxCtlFiGmMdZgxlpn5/P64t2lUZgzDuZ/xfj4e9+Gezzn3nvc9zbzvuZ97yFhrERERN/m8DiAiIqdPJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuAQVY0xhY8xsY0ySMWb6GTxPJ2PM/LzM5hVjTENjzAavc0hwMrpOXE6HMaYj0B24BDgErABettZ+fYbPexfwKFDfWpt2xkGDnDHGAhdaa3/2Oou4SWfikmvGmO7AMGAAEANUBEYBbfPg6SsBG8+HAj8VxphQrzNIkLPW6qbbKd+A4kAy0C6bbQriL/n4wG0YUDCw7jpgB9ADSAR+A+4LrOsLHAOOB/bxL+BFYHKW564MWCA0sHwvsAX/p4FfgE5Zxr/O8rj6wDIgKfBn/SzrFgL9gG8CzzMfiD7Ja/s9/1NZ8t8CtAI2AvuAXlm2rwssAQ4Eth0BhAXWfRV4LSmB19shy/M/DSQAk34fCzzmgsA+rgoslwN2A9d5/bOhmzc3nYlLbl0DFAI+zmab54B6wBXA5fiLrHeW9WXwvxnE4i/qkcaYSGttH/xn91OttRHW2vHZBTHGhAPDgZbW2qL4i3rF32xXEvg0sG0UMAT41BgTlWWzjsB9QGkgDOiZza7L4D8GscALwDjgTqAW0BB43hhTJbBtOvAEEI3/2N0AdAWw1jYKbHN54PVOzfL8JfF/KumcdcfW2s34C36yMaYI8C4w0Vq7MJu8ko+pxCW3ooA9Nvvpjk7AS9baRGvtbvxn2HdlWX88sP64tXYu/rPQi08zTwZwqTGmsLX2N2vtmr/Z5iZgk7V2krU2zVr7AbAeuDnLNu9aazdaa1OBafjfgE7mOP75/+PAFPwF/Ya19lBg/2vxv3lhrV1urf0usN+twFig8Sm8pj7W2qOBPCew1o4Dfga+B8rif9OU85RKXHJrLxCdw1xtOWBbluVtgbHM5/jTm8BhICK3Qay1KfinILoAvxljPjXGXHIKeX7PFJtlOSEXefZaa9MD938v2V1Z1qf+/nhjzEXGmDnGmARjzEH8nzSis3lugN3W2iM5bDMOuBR401p7NIdtJR9TiUtuLQGO4p8HPpl4/FMBv6sYGDsdKUCRLMtlsq601s6z1jbFf0a6Hn+55ZTn90w7TzNTbozGn+tCa20xoBdgcnhMtpeMGWMi8H/PMB54MTBdJOcplbjkirU2Cf888EhjzC3GmCLGmALGmJbGmIGBzT4AehtjShljogPbTz7NXa4AGhljKhpjigPP/r7CGBNjjGkbmBs/in9aJuNvnmMucJExpqMxJtQY0wGoDsw5zUy5URQ4CCQHPiU89Kf1u4CquXzON4A4a+2/8c/1jznjlOIslbjkmrV2MP5rxHvjvzLiV+AR4JPAJv2BOGAV8BPwQ2DsdPb1OTA18FzLObF4fYEc8fiv2GjMX0sSa+1eoDX+K2L24r+ypLW1ds/pZMqlnvi/ND2E/1PC1D+tfxGYaIw5YIxpn9OTGWPaAi3443V2B64yxnTKs8TiFP1lHxERh+lMXETEYSpxERGHqcRFRBymEhcRcdhZ/8d1LipVW9+c5pGdKefiYorzR6gvxOsI+Ur76Ku8jpCvjNs6Pae/TwDoTFxExGkqcRERh6nERUQcphIXEXGYSlxExGEqcRERh6nERUQcphIXEXGYSlxExGEqcRERh6nERUQcphIXEXGYSlxExGEqcRERh6nERUQcphIXEXGYSlxExGEqcRERh6nERUQcphIXEXGYSlxExGEqcRERh4V6HcArC5bPIiX5MBkZ6aSlpXNb07v5x6UX0XfQsxQsFEZaWjp9n3qNVT+u8TpqUBs9ZiAtWzRh9+691KnTPHO8S5d76Pzg3aSnpzPvvwvo3ftVD1O6Iza2LGPGvU7p0lFYa5nw7lTGjJpAv/7P0KJVE44dO84vv2zn4S5PkZR0yOu4QemegQ9Rs0ktDu1N4sXmPQC4+fF2NLzjRpL3HQTgo4H/YfXCH4kqX4qXvhjGri3xAGz5cSOTnxvnWfbTcd6WOMDdtz7I/n1JmctPvvAYI14fx1f/+5bGNzbgyT6PcdctD3qYMPhNnjSDsWMmMm7ckMyxRo2uoXXrptS7uiXHjh2jVKkoDxO6JS0tjd7PDmDlyjVERISzaPFMvlzwNV8u+JoX+wwiPT2dvi89RfceD9HnhYFexw1K385YyJcT/8v9Qx45YfyL8XOYP272X7bfvS2Bl1o9ea7i5TlNp2RhsUQUDQcgomgEiQm7PU4U/L75Zin7srwRAvz7gU4MHjyaY8eOAbB7914vojlp167drFzp//SXnJzChg0/U65sDAsWfE16ejoAy5atoFxsGS9jBrVNS9eRkpTsdYxzJsczcWPMJUBbIDYwtBOYZa1ddzaDnW3WWt6ZPhJrLVMnfsTUSR8z4LnBjJ82gqdf7IbP56NDq/u9jumkCy+sSv0Gdenz4pMcOXKUXr1e5oflq7yO5ZyKFWOpeXkN4uJWnjB+512389GHn3qUyl3X39OCa/6vMVt/2sz0/u9x+GAKANEVSvP8pwNJTU5l5usfsGnZeo+T5k62JW6MeRr4JzAFWBoYLg98YIyZYq11dqKzY+t/sythNyWjI5kwfSSbf95Ki5tvYMDzQ5g/ZwEt297IgGHPc+/tD3sd1TmhISFERhbnusa3UKv25UyaNJIa1Rt6Hcsp4eFFmPT+KJ59uh+HDv1xVtnzya6kpaczbepMD9O5Z+Hk+cwZ/iFYS9sed9Cu991MfGo0SYn7ebr+Q6QcSKbipVV5+K0n6dOsO0eSU72OfMpymk75F1DHWvuqtXZy4PYqUDew7m8ZYzobY+KMMXFJR4JzSmJXYKpk3579fD53ITWvrMGtHVozf84CAD6b+QU1r6rhZURn7YxPYNbMeQAsj1tJRkYG0dElPU7ljtDQUCa9P5JpU2cye9b8zPGOnW6jeYvreeD+JzxM56ZDe5KwGRlYa1k85QuqXF4NgLRjaaQc8L9Jbl+9hd3bdxFTpayXUXMtpxLPAMr9zXjZwLq/Za19y1pb21pbu3ihUmeS76woXKQQ4eFFMu83uO5qNq3fTGLCburWrwXANQ3rsHXLr17GdNbs2fNp1LgeANWqVSEsrAB79uzzOJU7Rox6lQ0bNjNyxDuZYzfc2IhuTzzAHR0eJDX1iIfp3FS8VInM+1c2r8vOjf7f7YiSxTA+fw1GVyhN6cpl2b090ZOMpyunOfHHgf8ZYzYBvzdaRaAa8MhJHxXkoktFMXLCIABCQkOY/dE8Fi9YQu+U/jz3ck9CQ0I4evQYz3d/2eOkwW/ChOE0bFSPqKhINm5aQv/+Q3lv4jTGjBnIsmXzOHb8OJ0f6OF1TGfUu6YW/+x4K6tXr2fxt/4rKV56cTADB71AWMEwPpk1EYC4ZSt4otvzXkYNWg8M78ZF9WoQEVmUgUvGMGvoNC6qV4MK1SuDtezZsZvJvcYCcFHdf9C2ewfS09LJyMhg8nNvcdixL0WNtTb7DYzx4Z8+yfrF5jJrbfqp7OCiUrWz34Gcsp0pe7yOkK+E+kK8jpCvtI++yusI+cq4rdPNqWyX49Up1toM4LszTiQiInlO14mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDgsx//b/ZnakvTb2d7FeSM1frHXEfKVmCrNvY6Qr1ycUdDrCOclnYmLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw0K9DhAMmje7jiFDXiLE5+Oddz9g4KCRXkcKes1uu4fwIkXw+XyEhIQw7Z3hrN+0hX6D3uRw6hHKlS3Na32eIiI8nJ/WbuDF14YDYLF0vb8TNzZu4PErCE6xsWUY9dYgSpeOxlrLxHenMnb0RMZPGEa1C6sCULx4UZKSDtG4QRuP0wafomVL0nJoF8JLFcday6r/fMkP78yjUPFwWo96hOLlS5G0Yzezu77J0aTDhBUtzE1vPETRclH4QkOIGzuX1dO/8vpl5Iqx1p7VHYSGxZ7dHZwhn8/HujWLadHqn+zY8RvfLZnLnXd1Zd26TV5H+4vU+MVeR8jU7LZ7mDp+OJElimeOdfjXY/R85N/UubImH82Zx874XTza+W5SjxyhQGgBQkND2L1nH7fd05UFM98nNDTEw1cAMVWae7r/vxMTU4qYMqVYtXItERHhLFj8MXfd0ZUNG37O3KbfgGc4mJTMoNdGeJj0r3pFXu11BMJLlyC8dAkSV2+lQHgh7vq0HzMfGEqNdo04ciCFpaNmU7frzRQqXoSvXpnK1Q+3oWCxwnz1ylQKlyzK/QsHMbrWw2QcT/f6pdBz+2RzKtvlejrFGPNe7uMEr7p1rmTz5q388st2jh8/zrRpM2lzc/D9crtg2687qX3FZQBcU+cqPl/0NQCFCxXKLOyjx46BOaWfzfPSrl27WbVyLQDJySls3LCZsuViTtjmlltb8eGM2V7EC3opiQdIXL0VgOMpR9j3czwRZUpSrWkt1szwnwStmbGYas1qA/5PhmHhhQEICy/EkQMpZKRleJL9dGU7nWKMmfXnIeB6Y0wJAGut85/nysWW4dcd8ZnLO3b+Rt06V3qYyA3GGDo/8RzGGNq1bUm7tq24oEolFixewg2N6jP/y8Uk7NqTuf2qNet5fsBQ4ncl8srzPT0/C3dBhYqx1KxZneVxKzPHrmlQh8TEPWzZvM3DZG4oVj6a0jUq8duPmykSXYyUxAOAv+iLRBcD4McJn3Pr+O50iRtBWHgh5jw8As7y7ERey2lOvDywFngbsPhLvDYwOLsHGWM6A50BTEhxfL7wM08qQeW90a8TUyqavfsP8MDjvahSqQL9ej3BK0NHM3bCB1x3bT0KFPjjx6tmjUuY+f5YNm/dznP9B9OwXh0KFgzz8BUEt/DwIkycPIJez7zMoUPJmeO33d6aj2bM8TCZGwoUKUibsd34su9kjiWnnnS7Ko0vI3HtNqbdMYASlWJo9/7T7Fi6IdvHBJucplNqA8uB54Aka+1CINVau8hau+hkD7LWvmWtrW2trR3sBR6/M4EK5ctlLpePLUt8fIKHidwQUyoagKjIEtzQqD4/rd1A1UoVGDdsANPeeZNWNzamQmzZvzzugsoVKVK4MJu2bD3Hid0RGhrKxMkjmDFtFnNmzc8cDwkJoXWbZnz84VwP0wU/X2gIbcZ2Y93H37Lpv3EAHN5zkPDSJQD/vPnhPQcBuLRd48xtDmzbRdKvuyl5wV9/boNZtiVurc2w1g4F7gOeM8aMIJ9d0bIsbgXVqlWhcuUKFChQgPbt2zJ7zvycH3geO5x6hJSUw5n3v136AxdWrcze/f6PqxkZGYydOIX2t7QCYEd8Amlp/i+K4hN28cu2X4ktG/P3Ty4MHzmAjRs2M2rEuyeMX3d9fTZt3KKTjBw0H/Rv9v0cz/K3P8sc2/z5D9S4vSEANW5vyM+fLwfgYPweKjWoAUCR6GJEXlCWpO2J5z70GTilQrbW7gDaGWNuAg6e3UjnVnp6Ot0e783cT/9DiM/HhIlTWbt2o9exgtreffvp1qsfAOlp6bRqdh3X1qvNpGmfMOUj/0f9GxvX59abmgHww6o1jJ80jdDQUHw+Q++eD59wVYv84epranFHx1tZs3o9i77xfyXVr+9gvpi/iFtvb82H0zWVkp3YOhdR47aG7F63nbs/exmAxQOn8f2o2dw8+lEu69CYgzv3MPuhNwFYMvwTWg5+kHvmv4Ix8NUrU0ndn5zdLoLOeX+JoUuC6RLD/CAYLzF0WTBcYpifnLVLDEVEJHioxEVEHKYSFxFxmEpcRMRhKnEREYepxEVEHKYSFxFxmEpcRMRhKnEREYepxEVEHKYSFxFxmEpcRMRhKnEREYepxEVEHKYSFxFxmEpcRMRhKnEREYepxEVEHKYSFxFxmEpcRMRhKnEREYepxEVEHBbqdQA5dSUqNvE6Qr6yb9l4ryPkKz1uesvrCOclnYmLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4rBQrwMEg+bNrmPIkJcI8fl4590PGDhopNeRnDF6zEBatmjC7t17qVOnOQC9nnuc++67gz179gHwYp+BzJu30MOUwa/Fg70pUrgQIT4fISE+pgx6hlFT5vDRF98QWawoAI91akPDWpcCsHHrDl4a8wEpqUcwxvDBwKcpGFbAy5cQNDoN7MKlTa7i0N6DDGjeM3O88T0taHh3M2x6BqsX/MjMV9/PXBdZLorenw9h7rDp/G/cHC9in7bzvsR9Ph/D33iZFq3+yY4dv/HdkrnMnjOfdes2eR3NCZMnzWDsmImMGzfkhPERb47njTfGeZTKTeNfepzIYhEnjN3Zugn33tL0hLG09HSefWMCAx67l4urlOfAoWRCQ0LOZdSg9t2MRSyaOI+7hzycOXbhNTW4rGltXm35FGnH0oiIKnbCY/6v992sWbjiXEfNE+f9dErdOleyefNWfvllO8ePH2fatJm0ubm517Gc8c03S9m3L8nrGOeVJSvWcVGlWC6uUh6AEkUjCAk573+VM21euo7DScknjDXs1JTPR88k7VgaAMl7D2auq9msNnt/TSRh06/nNGdeOe//y5eLLcOvO+Izl3fs/I1y5cp4mCh/eLDLPXz//WeMHjOQEiWK5fyA850xPNj3TTr0fIUZ87/OHJ7y2SJue6I/L4yYxMHkwwBsjU/EGEOXl96kfY9XeOfj+V6ldkbpqmW5oO4l9PykP92m9qFizQsACCtSkKZd2jL3jRkeJzx9Z6XEjTGdjTFxxpi4jIyUs7ELCWJvj5vMpTUaUa9eKxISEnnl1d5eRwp6E1/uwbTBzzKq9yNM+WwRcWs20aFFIz4d9RLTB/ciOrIYr0/4EID09HR+WLeZVx6/j4kDerDg+5V8t2q9x68guPlCQggvHsHrt/TmkwGTuX/k4wDc9Hg7Foz/lGOHj3qc8PRlW+LGmOLGmFeNMeuNMfuMMXuNMesCYyVO9jhr7VvW2trW2to+X3jep85D8TsTqFC+XOZy+diyxMcneJjIfYmJe8jIyMBay7vvTKF2rcu9jhT0YqL8v05RJYrS5OrLWb1pK1ElihES4sPn83Fb02v5adNW/7bRkdSqXo3IYhEULhhGw6tqsG6Lm1MB58qBhL2smLcUgG0rN2MzMogoWZRKV1Tjlmc70ffrN7nu/lY0e/hWGt3t1nRqTmfi04D9wHXW2pLW2ijg+sDYtLMd7lxYFreCatWqULlyBQoUKED79m2ZPUcfT89EmTKlMu+3adOcNWs3epgm+B0+cpSU1COZ95esXEe1iuXYneW7hgXfr+DCiv6TjQZXVGfTtnhSjx4jLT2duLWbuKC8pgCzs2r+Mi6qVx2A0lXKEloglOR9hxjW/kX6XPsofa59lIXvzGX+yI/56r15HqfNnZyuTqlsrX0t64C1NgF4zRhz/9mLde6kp6fT7fHezP30P4T4fEyYOJW1Kp1TNmHCcBo2qkdUVCQbNy2hf/+hNGpYj5o1q2OtZdv2HTz2aC+vYwa1fQcO8fhrYwFIz8igZcPaXHtVDXq9MYH1v+zAGChXKooXunQEoFhEEe5u04SOT/l/NRvWqkGj2pd5lj/Y3Dv8MS6sV52IyKL0WzKKuUOns2Tal3Qa+BC95r1O+vE0JvUY5XXMPGOstSdfacx84AtgorV2V2AsBrgXaGqtvTGnHYSGxZ58B5IrBUN1HXBe2rdsvNcR8pUeN73ldYR8ZcTWqeZUtstpOqUDEAUsCsyJ7wMWAiWBdmeUUEREzli20ynW2v3A04HbCYwx9wHvnqVcIiJyCs7kEsO+eZZCREROS7Zn4saYVSdbBcTkfRwREcmNnK5OiQGa47+kMCsDfHtWEomIyCnLqcTnABHW2r/8yzDGmIVnJZGIiJyynL7Y/Fc26zrmfRwREcmN8/4fwBIRcZlKXETEYSpxERGHqcRFRBymEhcRcZhKXETEYSpxERGHqcRFRBymEhcRcZhKXETEYSpxERGHqcRFRBymEhcRcZhKXETEYSpxERGHGWvtWd1BaFjs2d3BeaR7uUZeR8hXttlUryPkK5O+1v92Ny+FVbjcnMp2OhMXEXGYSlxExGEqcRERh6nERUQcphIXEXGYSlxExGEqcRERh6nERUQcphIXEXGYSlxExGEqcRERh6nERUQcphIXEXGYSlxExGEqcRERh6nERUQcphIXEXGYSlxExGEqcRERh6nERUQcphIXEXGYSlxExGGhXgfwWvny5ZjwzhuUjonGWsvbb7/PmyPGex0rqLUb+CD/aHIlyXsPMqT5UwB0GvEYpaqWBaBQsXCOHExhWKtn8YWGcPtrnYmtURlfaAg/fLSYL0fN9DJ+0Oky6BGualKbg3uT6NmsGwDte3SkdtO62AxL0t4kRvd4g/2J+6ndtC7te3TEZljS09OZ2Hc8G+LWefwKgkvzTg9TpHAhQkJ8hISEMHXUq/TsN5StO+IBOJR8mKIRRZgxdhDHj6fRd9hbrNmwGZ/PxzNd76XOFTU8fgW5c96XeFpaGk8+1ZcfV6wmIiKcpd//ly/+9xXr1m3yOlrQipuxiG8nzqPDkK6ZY+8/Mjzzfuvn7uTIocMA1Gx1NaFhoQxt8TQFCoXR44vXWTHrG/bv2HPOcwerRdMXMG/iXB4e0i1zbPbYj5k2+D8AtLj3Jm7r1oG3nxvDT9+sIu7zpQBUvKQSj498ku43POJJ7mD2zuA+RBYvlrn8+vNPZN4fNOY9IsKLADBj7hcAfPz2YPbuT+KhXgOYMvIVfD53JincSXqWJCQk8uOK1QAkJ6ewfv0mYsuV8ThVcPtl6XoOJyWfdH3Nm+qxYta3mcthhQviC/FRoFAY6cfSOHIo9VzEdMa6pWtJPnDi8UxN/uMYFSpSCGstAEcPH8kcL1ikEGDPScb8wlrLvEVLaHV9AwA2b9vB1VdcCkBUZHGKRYSzZuMWLyPm2nl/Jp5VpUrlueLyS/l+6Y9eR3FWlbqXkLwniT1bEwBYNfd7qjetRe+lowkrHMbsfpNITUrxOKUbOjzZiUb/dz2ph1Loe8fzmeN1ml/NP5+6i+LRxXn1vv4eJgxOxsCDT78MBtrd1JR2rW/MXLf8p3VERRanUnn/1N/FVSvz5ZI4WjZpQELiXtZu3EJC4h4uu6SaV/FzLdclbowpba1NPBthvBQeXoRpU8fRvWcfDh06+VmmZO+KNvVPOAuvcPkF2PQM+l/dlcLFw+k6rQ+bvl7Nvl/z3Y9Qnps66H2mDnqfW7reRot7WjF96BQAls37nmXzvucfdavToUdH+nfq43HS4DJxWD9iokuyd38SnZ/uT5WK5ahdszoAny34JvMsHODWltezZfsO7uj6DGVLl+LyGhfjC3FrgiLbtMaYkn+6RQFLjTGRxpiS2TyuszEmzhgTl5ER/GddoaGhTJ86jg8++JhPPvnM6zjO8oX4uLR5XVbOWZI5dmXbBmxYtJKMtHRS9h5k6/KNlK9Z1cOU7ln8ySKubnnNX8bXLV1L6YoxFI0s6kGq4BUT7a+mqMji3NCgDqvX/wxAWno6X3y9lObX1c/cNjQkhKe73suMsYN4s99THEpOoXL5cp7kPl05veXsAZZnucUBscAPgft/y1r7lrW2trW2ts8XnldZz5pxbw1m3fqfGfbGW15HcVq1ay9j95Z4khL2ZY4diN/DBfX93/YXKFyQildWI3FzvFcRnVGmctnM+3WaXc3OzTsBiKn0x/c1VS6tSoGwAhzaf+ic5wtWh1OPkHI4NfP+t8tXUa1yRQC+W/4TVSqWo0ypqMztU48c5XCq/3uGb5evIiQkhAsqlT/3wc9ATtMpTwJNgSettT8BGGN+sdZWOevJzpEG9etw1523s+qntcQtmw/A88+/ymf/XeBxsuDVcfijVK33D8Iji9JryQg+HzqDZdMWcsXN15wwlQLw7XvzaT+oC93nD8IYiJu+iIT12z1KHpweG96d6tdcStHIYoz67m2mD53CldfXolzVcmRkWPbs3M24XqMBuLrlNTS67XrSj6dz7OhRhj38usfpg8ve/Uk8/qL/mKSnp9OqybVcW/cKAD5beOJUCsC+A0l0eeZljM9H6aiSvPKMe1f6mN+/9T7pBsaUB4YCvwJ9gJXW2lP+PBwaFquvz/NI93KNvI6Qr2yzukomL036uq/XEfKVsAqXm1PZLscZfGvtDmttO2Ah8DlQ5MyiiYhIXjnlr2GttbOA64EbAYwx952tUCIicmpydS2NtTbVWrs6sKjPTiIiHsv2i01jzKqTrQJi8j6OiIjkRk5Xp8QAzYH9fxo3wLd/3VxERM6lnEp8DhBhrV3x5xXGmIVnJZGIiJyybEvcWvuvbNZ1zPs4IiKSG279IwEiInIClbiIiMNU4iIiDlOJi4g4TCUuIuIwlbiIiMNU4iIiDlOJi4g4TCUuIuIwlbiIiMNU4iIiDlOJi4g4TCUuIuIwlbiIiMNU4iIiDlOJi4g4zFhrvc4QFIwxna21b3mdI7/Q8cw7OpZ5K78dT52J/6Gz1wHyGR3PvKMOjgMbAAACOElEQVRjmbfy1fFUiYuIOEwlLiLiMJX4H/LNHFmQ0PHMOzqWeStfHU99sSki4jCdiYuIOEwlLiLiMJU4YIxpYYzZYIz52RjzjNd5XGeMCTHG/GiMmeN1FtcYY94xxiQaY1ZnGStpjPncGLMp8Geklxld8nfHM8u6HsYYa4yJ9iJbXjnvS9wYEwKMBFoC1YF/GmOqe5vKed2AdV6HcNQEoMWfxp4B/metvRD4X2BZTs0E/no8McZUAJoB2891oLx23pc4UBf42Vq7xVp7DJgCtPU4k7OMMeWBm4C3vc7iImvtV8C+Pw23BSYG7k8EbjmnoRx2kuMJMBR4CnD+yg6VOMQCv2ZZ3hEYk9MzDP8vR4bXQfKRGGvtb4H7CUCMl2FcZ4xpC+y01q70OkteUIlLnjHGtAYSrbXLvc6SX1n/NcHOnz16xRhTBOgFvOB1lryiEoedQIUsy+UDY5J7DYA2xpit+KelmhhjJnsbKV/YZYwpCxD4M9HjPC67AKgCrAz8nJYHfjDGlPE01RlQicMy4EJjTBVjTBhwBzDL40xOstY+a60tb62tjP84LrDW3ulxrPxgFnBP4P49wEwPszjNWvuTtba0tbZy4Od0B3CVtTbB42in7bwvcWttGvAIMA//FRXTrLVrvE0l5ytjzAfAEuBiY8wOY8y/gFeBpsaYTcCNgWU5BSc5nvmK/tq9iIjDzvszcRERl6nERUQcphIXEXGYSlxExGEqcRERh6nERUQcphIXEXHY/wNyP9u5CfZxtQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "print_result(logreg, test_text, X_test, y_test, cv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tuned hpyerparameters :(best parameters)  {'loss': 'hinge', 'penalty': 'l2'}\n",
      "accuracy : 0.7748042963772074\n"
     ]
    }
   ],
   "source": [
    "param = [\n",
    "    {\"loss\":['hinge','squared_hinge'], 'penalty': ['l2']},\n",
    "]\n",
    "model = LinearSVC()\n",
    "cv = GridSearchCV(model, param, cv = 3, n_jobs = 7)\n",
    "svc = Pipeline([\n",
    "                    ('tfidf', vec_model),\n",
    "                    ('clf', cv),\n",
    "               ])\n",
    "svc.fit(X_train, y_train)\n",
    "print(\"tuned hpyerparameters :(best parameters) \", cv.best_params_)\n",
    "print(\"accuracy :\", cv.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted service ID:  [0]\n",
      "Precision:   0.75\n",
      "Recall:   0.59\n",
      "F1-measure:   0.61\n",
      "Accuracy:   0.66\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.97      0.23      0.37       251\n",
      "           4       0.74      0.72      0.73       828\n",
      "          10       0.75      0.76      0.75       735\n",
      "          14       0.52      0.65      0.58       892\n",
      "\n",
      "   micro avg       0.66      0.66      0.66      2706\n",
      "   macro avg       0.75      0.59      0.61      2706\n",
      "weighted avg       0.69      0.66      0.65      2706\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXEAAAEICAYAAACpqsStAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3XmcTfXjx/HX584Yy4xlzDAYe1q+lBZLIpSyJupX6Ev795ukRaFNSiIVWZI1KaJvthZL+lJfkUoxCtlFiGmMdZgxlpn5/P64t2lUZgzDuZ/xfj4e9+Gezzn3nvc9zbzvuZ97yFhrERERN/m8DiAiIqdPJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuAQVY0xhY8xsY0ySMWb6GTxPJ2PM/LzM5hVjTENjzAavc0hwMrpOXE6HMaYj0B24BDgErABettZ+fYbPexfwKFDfWpt2xkGDnDHGAhdaa3/2Oou4SWfikmvGmO7AMGAAEANUBEYBbfPg6SsBG8+HAj8VxphQrzNIkLPW6qbbKd+A4kAy0C6bbQriL/n4wG0YUDCw7jpgB9ADSAR+A+4LrOsLHAOOB/bxL+BFYHKW564MWCA0sHwvsAX/p4FfgE5Zxr/O8rj6wDIgKfBn/SzrFgL9gG8CzzMfiD7Ja/s9/1NZ8t8CtAI2AvuAXlm2rwssAQ4Eth0BhAXWfRV4LSmB19shy/M/DSQAk34fCzzmgsA+rgoslwN2A9d5/bOhmzc3nYlLbl0DFAI+zmab54B6wBXA5fiLrHeW9WXwvxnE4i/qkcaYSGttH/xn91OttRHW2vHZBTHGhAPDgZbW2qL4i3rF32xXEvg0sG0UMAT41BgTlWWzjsB9QGkgDOiZza7L4D8GscALwDjgTqAW0BB43hhTJbBtOvAEEI3/2N0AdAWw1jYKbHN54PVOzfL8JfF/KumcdcfW2s34C36yMaYI8C4w0Vq7MJu8ko+pxCW3ooA9Nvvpjk7AS9baRGvtbvxn2HdlWX88sP64tXYu/rPQi08zTwZwqTGmsLX2N2vtmr/Z5iZgk7V2krU2zVr7AbAeuDnLNu9aazdaa1OBafjfgE7mOP75/+PAFPwF/Ya19lBg/2vxv3lhrV1urf0usN+twFig8Sm8pj7W2qOBPCew1o4Dfga+B8rif9OU85RKXHJrLxCdw1xtOWBbluVtgbHM5/jTm8BhICK3Qay1KfinILoAvxljPjXGXHIKeX7PFJtlOSEXefZaa9MD938v2V1Z1qf+/nhjzEXGmDnGmARjzEH8nzSis3lugN3W2iM5bDMOuBR401p7NIdtJR9TiUtuLQGO4p8HPpl4/FMBv6sYGDsdKUCRLMtlsq601s6z1jbFf0a6Hn+55ZTn90w7TzNTbozGn+tCa20xoBdgcnhMtpeMGWMi8H/PMB54MTBdJOcplbjkirU2Cf888EhjzC3GmCLGmALGmJbGmIGBzT4AehtjShljogPbTz7NXa4AGhljKhpjigPP/r7CGBNjjGkbmBs/in9aJuNvnmMucJExpqMxJtQY0wGoDsw5zUy5URQ4CCQHPiU89Kf1u4CquXzON4A4a+2/8c/1jznjlOIslbjkmrV2MP5rxHvjvzLiV+AR4JPAJv2BOGAV8BPwQ2DsdPb1OTA18FzLObF4fYEc8fiv2GjMX0sSa+1eoDX+K2L24r+ypLW1ds/pZMqlnvi/ND2E/1PC1D+tfxGYaIw5YIxpn9OTGWPaAi3443V2B64yxnTKs8TiFP1lHxERh+lMXETEYSpxERGHqcRFRBymEhcRcdhZ/8d1LipVW9+c5pGdKefiYorzR6gvxOsI+Ur76Ku8jpCvjNs6Pae/TwDoTFxExGkqcRERh6nERUQcphIXEXGYSlxExGEqcRERh6nERUQcphIXEXGYSlxExGEqcRERh6nERUQcphIXEXGYSlxExGEqcRERh6nERUQcphIXEXGYSlxExGEqcRERh6nERUQcphIXEXGYSlxExGEqcRERh4V6HcArC5bPIiX5MBkZ6aSlpXNb07v5x6UX0XfQsxQsFEZaWjp9n3qNVT+u8TpqUBs9ZiAtWzRh9+691KnTPHO8S5d76Pzg3aSnpzPvvwvo3ftVD1O6Iza2LGPGvU7p0lFYa5nw7lTGjJpAv/7P0KJVE44dO84vv2zn4S5PkZR0yOu4QemegQ9Rs0ktDu1N4sXmPQC4+fF2NLzjRpL3HQTgo4H/YfXCH4kqX4qXvhjGri3xAGz5cSOTnxvnWfbTcd6WOMDdtz7I/n1JmctPvvAYI14fx1f/+5bGNzbgyT6PcdctD3qYMPhNnjSDsWMmMm7ckMyxRo2uoXXrptS7uiXHjh2jVKkoDxO6JS0tjd7PDmDlyjVERISzaPFMvlzwNV8u+JoX+wwiPT2dvi89RfceD9HnhYFexw1K385YyJcT/8v9Qx45YfyL8XOYP272X7bfvS2Bl1o9ea7i5TlNp2RhsUQUDQcgomgEiQm7PU4U/L75Zin7srwRAvz7gU4MHjyaY8eOAbB7914vojlp167drFzp//SXnJzChg0/U65sDAsWfE16ejoAy5atoFxsGS9jBrVNS9eRkpTsdYxzJsczcWPMJUBbIDYwtBOYZa1ddzaDnW3WWt6ZPhJrLVMnfsTUSR8z4LnBjJ82gqdf7IbP56NDq/u9jumkCy+sSv0Gdenz4pMcOXKUXr1e5oflq7yO5ZyKFWOpeXkN4uJWnjB+512389GHn3qUyl3X39OCa/6vMVt/2sz0/u9x+GAKANEVSvP8pwNJTU5l5usfsGnZeo+T5k62JW6MeRr4JzAFWBoYLg98YIyZYq11dqKzY+t/sythNyWjI5kwfSSbf95Ki5tvYMDzQ5g/ZwEt297IgGHPc+/tD3sd1TmhISFERhbnusa3UKv25UyaNJIa1Rt6Hcsp4eFFmPT+KJ59uh+HDv1xVtnzya6kpaczbepMD9O5Z+Hk+cwZ/iFYS9sed9Cu991MfGo0SYn7ebr+Q6QcSKbipVV5+K0n6dOsO0eSU72OfMpymk75F1DHWvuqtXZy4PYqUDew7m8ZYzobY+KMMXFJR4JzSmJXYKpk3579fD53ITWvrMGtHVozf84CAD6b+QU1r6rhZURn7YxPYNbMeQAsj1tJRkYG0dElPU7ljtDQUCa9P5JpU2cye9b8zPGOnW6jeYvreeD+JzxM56ZDe5KwGRlYa1k85QuqXF4NgLRjaaQc8L9Jbl+9hd3bdxFTpayXUXMtpxLPAMr9zXjZwLq/Za19y1pb21pbu3ihUmeS76woXKQQ4eFFMu83uO5qNq3fTGLCburWrwXANQ3rsHXLr17GdNbs2fNp1LgeANWqVSEsrAB79uzzOJU7Rox6lQ0bNjNyxDuZYzfc2IhuTzzAHR0eJDX1iIfp3FS8VInM+1c2r8vOjf7f7YiSxTA+fw1GVyhN6cpl2b090ZOMpyunOfHHgf8ZYzYBvzdaRaAa8MhJHxXkoktFMXLCIABCQkOY/dE8Fi9YQu+U/jz3ck9CQ0I4evQYz3d/2eOkwW/ChOE0bFSPqKhINm5aQv/+Q3lv4jTGjBnIsmXzOHb8OJ0f6OF1TGfUu6YW/+x4K6tXr2fxt/4rKV56cTADB71AWMEwPpk1EYC4ZSt4otvzXkYNWg8M78ZF9WoQEVmUgUvGMGvoNC6qV4MK1SuDtezZsZvJvcYCcFHdf9C2ewfS09LJyMhg8nNvcdixL0WNtTb7DYzx4Z8+yfrF5jJrbfqp7OCiUrWz34Gcsp0pe7yOkK+E+kK8jpCvtI++yusI+cq4rdPNqWyX49Up1toM4LszTiQiInlO14mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDgsx//b/ZnakvTb2d7FeSM1frHXEfKVmCrNvY6Qr1ycUdDrCOclnYmLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw0K9DhAMmje7jiFDXiLE5+Oddz9g4KCRXkcKes1uu4fwIkXw+XyEhIQw7Z3hrN+0hX6D3uRw6hHKlS3Na32eIiI8nJ/WbuDF14YDYLF0vb8TNzZu4PErCE6xsWUY9dYgSpeOxlrLxHenMnb0RMZPGEa1C6sCULx4UZKSDtG4QRuP0wafomVL0nJoF8JLFcday6r/fMkP78yjUPFwWo96hOLlS5G0Yzezu77J0aTDhBUtzE1vPETRclH4QkOIGzuX1dO/8vpl5Iqx1p7VHYSGxZ7dHZwhn8/HujWLadHqn+zY8RvfLZnLnXd1Zd26TV5H+4vU+MVeR8jU7LZ7mDp+OJElimeOdfjXY/R85N/UubImH82Zx874XTza+W5SjxyhQGgBQkND2L1nH7fd05UFM98nNDTEw1cAMVWae7r/vxMTU4qYMqVYtXItERHhLFj8MXfd0ZUNG37O3KbfgGc4mJTMoNdGeJj0r3pFXu11BMJLlyC8dAkSV2+lQHgh7vq0HzMfGEqNdo04ciCFpaNmU7frzRQqXoSvXpnK1Q+3oWCxwnz1ylQKlyzK/QsHMbrWw2QcT/f6pdBz+2RzKtvlejrFGPNe7uMEr7p1rmTz5q388st2jh8/zrRpM2lzc/D9crtg2687qX3FZQBcU+cqPl/0NQCFCxXKLOyjx46BOaWfzfPSrl27WbVyLQDJySls3LCZsuViTtjmlltb8eGM2V7EC3opiQdIXL0VgOMpR9j3czwRZUpSrWkt1szwnwStmbGYas1qA/5PhmHhhQEICy/EkQMpZKRleJL9dGU7nWKMmfXnIeB6Y0wJAGut85/nysWW4dcd8ZnLO3b+Rt06V3qYyA3GGDo/8RzGGNq1bUm7tq24oEolFixewg2N6jP/y8Uk7NqTuf2qNet5fsBQ4ncl8srzPT0/C3dBhYqx1KxZneVxKzPHrmlQh8TEPWzZvM3DZG4oVj6a0jUq8duPmykSXYyUxAOAv+iLRBcD4McJn3Pr+O50iRtBWHgh5jw8As7y7ERey2lOvDywFngbsPhLvDYwOLsHGWM6A50BTEhxfL7wM08qQeW90a8TUyqavfsP8MDjvahSqQL9ej3BK0NHM3bCB1x3bT0KFPjjx6tmjUuY+f5YNm/dznP9B9OwXh0KFgzz8BUEt/DwIkycPIJez7zMoUPJmeO33d6aj2bM8TCZGwoUKUibsd34su9kjiWnnnS7Ko0vI3HtNqbdMYASlWJo9/7T7Fi6IdvHBJucplNqA8uB54Aka+1CINVau8hau+hkD7LWvmWtrW2trR3sBR6/M4EK5ctlLpePLUt8fIKHidwQUyoagKjIEtzQqD4/rd1A1UoVGDdsANPeeZNWNzamQmzZvzzugsoVKVK4MJu2bD3Hid0RGhrKxMkjmDFtFnNmzc8cDwkJoXWbZnz84VwP0wU/X2gIbcZ2Y93H37Lpv3EAHN5zkPDSJQD/vPnhPQcBuLRd48xtDmzbRdKvuyl5wV9/boNZtiVurc2w1g4F7gOeM8aMIJ9d0bIsbgXVqlWhcuUKFChQgPbt2zJ7zvycH3geO5x6hJSUw5n3v136AxdWrcze/f6PqxkZGYydOIX2t7QCYEd8Amlp/i+K4hN28cu2X4ktG/P3Ty4MHzmAjRs2M2rEuyeMX3d9fTZt3KKTjBw0H/Rv9v0cz/K3P8sc2/z5D9S4vSEANW5vyM+fLwfgYPweKjWoAUCR6GJEXlCWpO2J5z70GTilQrbW7gDaGWNuAg6e3UjnVnp6Ot0e783cT/9DiM/HhIlTWbt2o9exgtreffvp1qsfAOlp6bRqdh3X1qvNpGmfMOUj/0f9GxvX59abmgHww6o1jJ80jdDQUHw+Q++eD59wVYv84epranFHx1tZs3o9i77xfyXVr+9gvpi/iFtvb82H0zWVkp3YOhdR47aG7F63nbs/exmAxQOn8f2o2dw8+lEu69CYgzv3MPuhNwFYMvwTWg5+kHvmv4Ix8NUrU0ndn5zdLoLOeX+JoUuC6RLD/CAYLzF0WTBcYpifnLVLDEVEJHioxEVEHKYSFxFxmEpcRMRhKnEREYepxEVEHKYSFxFxmEpcRMRhKnEREYepxEVEHKYSFxFxmEpcRMRhKnEREYepxEVEHKYSFxFxmEpcRMRhKnEREYepxEVEHKYSFxFxmEpcRMRhKnEREYepxEVEHBbqdQA5dSUqNvE6Qr6yb9l4ryPkKz1uesvrCOclnYmLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4rBQrwMEg+bNrmPIkJcI8fl4590PGDhopNeRnDF6zEBatmjC7t17qVOnOQC9nnuc++67gz179gHwYp+BzJu30MOUwa/Fg70pUrgQIT4fISE+pgx6hlFT5vDRF98QWawoAI91akPDWpcCsHHrDl4a8wEpqUcwxvDBwKcpGFbAy5cQNDoN7MKlTa7i0N6DDGjeM3O88T0taHh3M2x6BqsX/MjMV9/PXBdZLorenw9h7rDp/G/cHC9in7bzvsR9Ph/D33iZFq3+yY4dv/HdkrnMnjOfdes2eR3NCZMnzWDsmImMGzfkhPERb47njTfGeZTKTeNfepzIYhEnjN3Zugn33tL0hLG09HSefWMCAx67l4urlOfAoWRCQ0LOZdSg9t2MRSyaOI+7hzycOXbhNTW4rGltXm35FGnH0oiIKnbCY/6v992sWbjiXEfNE+f9dErdOleyefNWfvllO8ePH2fatJm0ubm517Gc8c03S9m3L8nrGOeVJSvWcVGlWC6uUh6AEkUjCAk573+VM21euo7DScknjDXs1JTPR88k7VgaAMl7D2auq9msNnt/TSRh06/nNGdeOe//y5eLLcOvO+Izl3fs/I1y5cp4mCh/eLDLPXz//WeMHjOQEiWK5fyA850xPNj3TTr0fIUZ87/OHJ7y2SJue6I/L4yYxMHkwwBsjU/EGEOXl96kfY9XeOfj+V6ldkbpqmW5oO4l9PykP92m9qFizQsACCtSkKZd2jL3jRkeJzx9Z6XEjTGdjTFxxpi4jIyUs7ELCWJvj5vMpTUaUa9eKxISEnnl1d5eRwp6E1/uwbTBzzKq9yNM+WwRcWs20aFFIz4d9RLTB/ciOrIYr0/4EID09HR+WLeZVx6/j4kDerDg+5V8t2q9x68guPlCQggvHsHrt/TmkwGTuX/k4wDc9Hg7Foz/lGOHj3qc8PRlW+LGmOLGmFeNMeuNMfuMMXuNMesCYyVO9jhr7VvW2trW2to+X3jep85D8TsTqFC+XOZy+diyxMcneJjIfYmJe8jIyMBay7vvTKF2rcu9jhT0YqL8v05RJYrS5OrLWb1pK1ElihES4sPn83Fb02v5adNW/7bRkdSqXo3IYhEULhhGw6tqsG6Lm1MB58qBhL2smLcUgG0rN2MzMogoWZRKV1Tjlmc70ffrN7nu/lY0e/hWGt3t1nRqTmfi04D9wHXW2pLW2ijg+sDYtLMd7lxYFreCatWqULlyBQoUKED79m2ZPUcfT89EmTKlMu+3adOcNWs3epgm+B0+cpSU1COZ95esXEe1iuXYneW7hgXfr+DCiv6TjQZXVGfTtnhSjx4jLT2duLWbuKC8pgCzs2r+Mi6qVx2A0lXKEloglOR9hxjW/kX6XPsofa59lIXvzGX+yI/56r15HqfNnZyuTqlsrX0t64C1NgF4zRhz/9mLde6kp6fT7fHezP30P4T4fEyYOJW1Kp1TNmHCcBo2qkdUVCQbNy2hf/+hNGpYj5o1q2OtZdv2HTz2aC+vYwa1fQcO8fhrYwFIz8igZcPaXHtVDXq9MYH1v+zAGChXKooXunQEoFhEEe5u04SOT/l/NRvWqkGj2pd5lj/Y3Dv8MS6sV52IyKL0WzKKuUOns2Tal3Qa+BC95r1O+vE0JvUY5XXMPGOstSdfacx84AtgorV2V2AsBrgXaGqtvTGnHYSGxZ58B5IrBUN1HXBe2rdsvNcR8pUeN73ldYR8ZcTWqeZUtstpOqUDEAUsCsyJ7wMWAiWBdmeUUEREzli20ynW2v3A04HbCYwx9wHvnqVcIiJyCs7kEsO+eZZCREROS7Zn4saYVSdbBcTkfRwREcmNnK5OiQGa47+kMCsDfHtWEomIyCnLqcTnABHW2r/8yzDGmIVnJZGIiJyynL7Y/Fc26zrmfRwREcmN8/4fwBIRcZlKXETEYSpxERGHqcRFRBymEhcRcZhKXETEYSpxERGHqcRFRBymEhcRcZhKXETEYSpxERGHqcRFRBymEhcRcZhKXETEYSpxERGHGWvtWd1BaFjs2d3BeaR7uUZeR8hXttlUryPkK5O+1v92Ny+FVbjcnMp2OhMXEXGYSlxExGEqcRERh6nERUQcphIXEXGYSlxExGEqcRERh6nERUQcphIXEXGYSlxExGEqcRERh6nERUQcphIXEXGYSlxExGEqcRERh6nERUQcphIXEXGYSlxExGEqcRERh6nERUQcphIXEXGYSlxExGGhXgfwWvny5ZjwzhuUjonGWsvbb7/PmyPGex0rqLUb+CD/aHIlyXsPMqT5UwB0GvEYpaqWBaBQsXCOHExhWKtn8YWGcPtrnYmtURlfaAg/fLSYL0fN9DJ+0Oky6BGualKbg3uT6NmsGwDte3SkdtO62AxL0t4kRvd4g/2J+6ndtC7te3TEZljS09OZ2Hc8G+LWefwKgkvzTg9TpHAhQkJ8hISEMHXUq/TsN5StO+IBOJR8mKIRRZgxdhDHj6fRd9hbrNmwGZ/PxzNd76XOFTU8fgW5c96XeFpaGk8+1ZcfV6wmIiKcpd//ly/+9xXr1m3yOlrQipuxiG8nzqPDkK6ZY+8/Mjzzfuvn7uTIocMA1Gx1NaFhoQxt8TQFCoXR44vXWTHrG/bv2HPOcwerRdMXMG/iXB4e0i1zbPbYj5k2+D8AtLj3Jm7r1oG3nxvDT9+sIu7zpQBUvKQSj498ku43POJJ7mD2zuA+RBYvlrn8+vNPZN4fNOY9IsKLADBj7hcAfPz2YPbuT+KhXgOYMvIVfD53JincSXqWJCQk8uOK1QAkJ6ewfv0mYsuV8ThVcPtl6XoOJyWfdH3Nm+qxYta3mcthhQviC/FRoFAY6cfSOHIo9VzEdMa6pWtJPnDi8UxN/uMYFSpSCGstAEcPH8kcL1ikEGDPScb8wlrLvEVLaHV9AwA2b9vB1VdcCkBUZHGKRYSzZuMWLyPm2nl/Jp5VpUrlueLyS/l+6Y9eR3FWlbqXkLwniT1bEwBYNfd7qjetRe+lowkrHMbsfpNITUrxOKUbOjzZiUb/dz2ph1Loe8fzmeN1ml/NP5+6i+LRxXn1vv4eJgxOxsCDT78MBtrd1JR2rW/MXLf8p3VERRanUnn/1N/FVSvz5ZI4WjZpQELiXtZu3EJC4h4uu6SaV/FzLdclbowpba1NPBthvBQeXoRpU8fRvWcfDh06+VmmZO+KNvVPOAuvcPkF2PQM+l/dlcLFw+k6rQ+bvl7Nvl/z3Y9Qnps66H2mDnqfW7reRot7WjF96BQAls37nmXzvucfdavToUdH+nfq43HS4DJxWD9iokuyd38SnZ/uT5WK5ahdszoAny34JvMsHODWltezZfsO7uj6DGVLl+LyGhfjC3FrgiLbtMaYkn+6RQFLjTGRxpiS2TyuszEmzhgTl5ER/GddoaGhTJ86jg8++JhPPvnM6zjO8oX4uLR5XVbOWZI5dmXbBmxYtJKMtHRS9h5k6/KNlK9Z1cOU7ln8ySKubnnNX8bXLV1L6YoxFI0s6kGq4BUT7a+mqMji3NCgDqvX/wxAWno6X3y9lObX1c/cNjQkhKe73suMsYN4s99THEpOoXL5cp7kPl05veXsAZZnucUBscAPgft/y1r7lrW2trW2ts8XnldZz5pxbw1m3fqfGfbGW15HcVq1ay9j95Z4khL2ZY4diN/DBfX93/YXKFyQildWI3FzvFcRnVGmctnM+3WaXc3OzTsBiKn0x/c1VS6tSoGwAhzaf+ic5wtWh1OPkHI4NfP+t8tXUa1yRQC+W/4TVSqWo0ypqMztU48c5XCq/3uGb5evIiQkhAsqlT/3wc9ATtMpTwJNgSettT8BGGN+sdZWOevJzpEG9etw1523s+qntcQtmw/A88+/ymf/XeBxsuDVcfijVK33D8Iji9JryQg+HzqDZdMWcsXN15wwlQLw7XvzaT+oC93nD8IYiJu+iIT12z1KHpweG96d6tdcStHIYoz67m2mD53CldfXolzVcmRkWPbs3M24XqMBuLrlNTS67XrSj6dz7OhRhj38usfpg8ve/Uk8/qL/mKSnp9OqybVcW/cKAD5beOJUCsC+A0l0eeZljM9H6aiSvPKMe1f6mN+/9T7pBsaUB4YCvwJ9gJXW2lP+PBwaFquvz/NI93KNvI6Qr2yzukomL036uq/XEfKVsAqXm1PZLscZfGvtDmttO2Ah8DlQ5MyiiYhIXjnlr2GttbOA64EbAYwx952tUCIicmpydS2NtTbVWrs6sKjPTiIiHsv2i01jzKqTrQJi8j6OiIjkRk5Xp8QAzYH9fxo3wLd/3VxERM6lnEp8DhBhrV3x5xXGmIVnJZGIiJyybEvcWvuvbNZ1zPs4IiKSG279IwEiInIClbiIiMNU4iIiDlOJi4g4TCUuIuIwlbiIiMNU4iIiDlOJi4g4TCUuIuIwlbiIiMNU4iIiDlOJi4g4TCUuIuIwlbiIiMNU4iIiDlOJi4g4zFhrvc4QFIwxna21b3mdI7/Q8cw7OpZ5K78dT52J/6Gz1wHyGR3PvKMOjgMbAAACOElEQVRjmbfy1fFUiYuIOEwlLiLiMJX4H/LNHFmQ0PHMOzqWeStfHU99sSki4jCdiYuIOEwlLiLiMJU4YIxpYYzZYIz52RjzjNd5XGeMCTHG/GiMmeN1FtcYY94xxiQaY1ZnGStpjPncGLMp8Geklxld8nfHM8u6HsYYa4yJ9iJbXjnvS9wYEwKMBFoC1YF/GmOqe5vKed2AdV6HcNQEoMWfxp4B/metvRD4X2BZTs0E/no8McZUAJoB2891oLx23pc4UBf42Vq7xVp7DJgCtPU4k7OMMeWBm4C3vc7iImvtV8C+Pw23BSYG7k8EbjmnoRx2kuMJMBR4CnD+yg6VOMQCv2ZZ3hEYk9MzDP8vR4bXQfKRGGvtb4H7CUCMl2FcZ4xpC+y01q70OkteUIlLnjHGtAYSrbXLvc6SX1n/NcHOnz16xRhTBOgFvOB1lryiEoedQIUsy+UDY5J7DYA2xpit+KelmhhjJnsbKV/YZYwpCxD4M9HjPC67AKgCrAz8nJYHfjDGlPE01RlQicMy4EJjTBVjTBhwBzDL40xOstY+a60tb62tjP84LrDW3ulxrPxgFnBP4P49wEwPszjNWvuTtba0tbZy4Od0B3CVtTbB42in7bwvcWttGvAIMA//FRXTrLVrvE0l5ytjzAfAEuBiY8wOY8y/gFeBpsaYTcCNgWU5BSc5nvmK/tq9iIjDzvszcRERl6nERUQcphIXEXGYSlxExGEqcRERh6nERUQcphIXEXHY/wNyP9u5CfZxtQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "print_result(svc, test_text, X_test, y_test, cv, False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tuned hpyerparameters :(best parameters)  {'alpha': 1.0, 'fit_prior': False}\n",
      "accuracy : 0.7596941561987984\n"
     ]
    }
   ],
   "source": [
    "param = [\n",
    "    {'alpha': [0.1, 1.0], 'fit_prior':[True, False]},\n",
    "]\n",
    "model = MultinomialNB()\n",
    "cv = GridSearchCV(model, param, cv = 3, n_jobs = 7)\n",
    "nb = Pipeline([\n",
    "                    ('tfidf', vec_model),\n",
    "                    ('clf', cv),\n",
    "               ])\n",
    "nb.fit(X_train, y_train)\n",
    "print(\"tuned hpyerparameters :(best parameters) \", cv.best_params_)\n",
    "print(\"accuracy :\", cv.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted service ID:  [0]\n",
      "Predic_proba :  [[0.74054425 0.01140127 0.08196786 0.16608662]]\n",
      "Precision:   0.75\n",
      "Recall:   0.59\n",
      "F1-measure:   0.61\n",
      "Accuracy:   0.66\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.97      0.23      0.37       251\n",
      "           4       0.74      0.72      0.73       828\n",
      "          10       0.75      0.76      0.75       735\n",
      "          14       0.52      0.65      0.58       892\n",
      "\n",
      "   micro avg       0.66      0.66      0.66      2706\n",
      "   macro avg       0.75      0.59      0.61      2706\n",
      "weighted avg       0.69      0.66      0.65      2706\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXEAAAEICAYAAACpqsStAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3XmcTfXjx/HX584Yy4xlzDAYe1q+lBZLIpSyJupX6Ev795ukRaFNSiIVWZI1KaJvthZL+lJfkUoxCtlFiGmMdZgxlpn5/P64t2lUZgzDuZ/xfj4e9+Gezzn3nvc9zbzvuZ97yFhrERERN/m8DiAiIqdPJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuAQVY0xhY8xsY0ySMWb6GTxPJ2PM/LzM5hVjTENjzAavc0hwMrpOXE6HMaYj0B24BDgErABettZ+fYbPexfwKFDfWpt2xkGDnDHGAhdaa3/2Oou4SWfikmvGmO7AMGAAEANUBEYBbfPg6SsBG8+HAj8VxphQrzNIkLPW6qbbKd+A4kAy0C6bbQriL/n4wG0YUDCw7jpgB9ADSAR+A+4LrOsLHAOOB/bxL+BFYHKW564MWCA0sHwvsAX/p4FfgE5Zxr/O8rj6wDIgKfBn/SzrFgL9gG8CzzMfiD7Ja/s9/1NZ8t8CtAI2AvuAXlm2rwssAQ4Eth0BhAXWfRV4LSmB19shy/M/DSQAk34fCzzmgsA+rgoslwN2A9d5/bOhmzc3nYlLbl0DFAI+zmab54B6wBXA5fiLrHeW9WXwvxnE4i/qkcaYSGttH/xn91OttRHW2vHZBTHGhAPDgZbW2qL4i3rF32xXEvg0sG0UMAT41BgTlWWzjsB9QGkgDOiZza7L4D8GscALwDjgTqAW0BB43hhTJbBtOvAEEI3/2N0AdAWw1jYKbHN54PVOzfL8JfF/KumcdcfW2s34C36yMaYI8C4w0Vq7MJu8ko+pxCW3ooA9Nvvpjk7AS9baRGvtbvxn2HdlWX88sP64tXYu/rPQi08zTwZwqTGmsLX2N2vtmr/Z5iZgk7V2krU2zVr7AbAeuDnLNu9aazdaa1OBafjfgE7mOP75/+PAFPwF/Ya19lBg/2vxv3lhrV1urf0usN+twFig8Sm8pj7W2qOBPCew1o4Dfga+B8rif9OU85RKXHJrLxCdw1xtOWBbluVtgbHM5/jTm8BhICK3Qay1KfinILoAvxljPjXGXHIKeX7PFJtlOSEXefZaa9MD938v2V1Z1qf+/nhjzEXGmDnGmARjzEH8nzSis3lugN3W2iM5bDMOuBR401p7NIdtJR9TiUtuLQGO4p8HPpl4/FMBv6sYGDsdKUCRLMtlsq601s6z1jbFf0a6Hn+55ZTn90w7TzNTbozGn+tCa20xoBdgcnhMtpeMGWMi8H/PMB54MTBdJOcplbjkirU2Cf888EhjzC3GmCLGmALGmJbGmIGBzT4AehtjShljogPbTz7NXa4AGhljKhpjigPP/r7CGBNjjGkbmBs/in9aJuNvnmMucJExpqMxJtQY0wGoDsw5zUy5URQ4CCQHPiU89Kf1u4CquXzON4A4a+2/8c/1jznjlOIslbjkmrV2MP5rxHvjvzLiV+AR4JPAJv2BOGAV8BPwQ2DsdPb1OTA18FzLObF4fYEc8fiv2GjMX0sSa+1eoDX+K2L24r+ypLW1ds/pZMqlnvi/ND2E/1PC1D+tfxGYaIw5YIxpn9OTGWPaAi3443V2B64yxnTKs8TiFP1lHxERh+lMXETEYSpxERGHqcRFRBymEhcRcdhZ/8d1LipVW9+c5pGdKefiYorzR6gvxOsI+Ur76Ku8jpCvjNs6Pae/TwDoTFxExGkqcRERh6nERUQcphIXEXGYSlxExGEqcRERh6nERUQcphIXEXGYSlxExGEqcRERh6nERUQcphIXEXGYSlxExGEqcRERh6nERUQcphIXEXGYSlxExGEqcRERh6nERUQcphIXEXGYSlxExGEqcRERh4V6HcArC5bPIiX5MBkZ6aSlpXNb07v5x6UX0XfQsxQsFEZaWjp9n3qNVT+u8TpqUBs9ZiAtWzRh9+691KnTPHO8S5d76Pzg3aSnpzPvvwvo3ftVD1O6Iza2LGPGvU7p0lFYa5nw7lTGjJpAv/7P0KJVE44dO84vv2zn4S5PkZR0yOu4QemegQ9Rs0ktDu1N4sXmPQC4+fF2NLzjRpL3HQTgo4H/YfXCH4kqX4qXvhjGri3xAGz5cSOTnxvnWfbTcd6WOMDdtz7I/n1JmctPvvAYI14fx1f/+5bGNzbgyT6PcdctD3qYMPhNnjSDsWMmMm7ckMyxRo2uoXXrptS7uiXHjh2jVKkoDxO6JS0tjd7PDmDlyjVERISzaPFMvlzwNV8u+JoX+wwiPT2dvi89RfceD9HnhYFexw1K385YyJcT/8v9Qx45YfyL8XOYP272X7bfvS2Bl1o9ea7i5TlNp2RhsUQUDQcgomgEiQm7PU4U/L75Zin7srwRAvz7gU4MHjyaY8eOAbB7914vojlp167drFzp//SXnJzChg0/U65sDAsWfE16ejoAy5atoFxsGS9jBrVNS9eRkpTsdYxzJsczcWPMJUBbIDYwtBOYZa1ddzaDnW3WWt6ZPhJrLVMnfsTUSR8z4LnBjJ82gqdf7IbP56NDq/u9jumkCy+sSv0Gdenz4pMcOXKUXr1e5oflq7yO5ZyKFWOpeXkN4uJWnjB+512389GHn3qUyl3X39OCa/6vMVt/2sz0/u9x+GAKANEVSvP8pwNJTU5l5usfsGnZeo+T5k62JW6MeRr4JzAFWBoYLg98YIyZYq11dqKzY+t/sythNyWjI5kwfSSbf95Ki5tvYMDzQ5g/ZwEt297IgGHPc+/tD3sd1TmhISFERhbnusa3UKv25UyaNJIa1Rt6Hcsp4eFFmPT+KJ59uh+HDv1xVtnzya6kpaczbepMD9O5Z+Hk+cwZ/iFYS9sed9Cu991MfGo0SYn7ebr+Q6QcSKbipVV5+K0n6dOsO0eSU72OfMpymk75F1DHWvuqtXZy4PYqUDew7m8ZYzobY+KMMXFJR4JzSmJXYKpk3579fD53ITWvrMGtHVozf84CAD6b+QU1r6rhZURn7YxPYNbMeQAsj1tJRkYG0dElPU7ljtDQUCa9P5JpU2cye9b8zPGOnW6jeYvreeD+JzxM56ZDe5KwGRlYa1k85QuqXF4NgLRjaaQc8L9Jbl+9hd3bdxFTpayXUXMtpxLPAMr9zXjZwLq/Za19y1pb21pbu3ihUmeS76woXKQQ4eFFMu83uO5qNq3fTGLCburWrwXANQ3rsHXLr17GdNbs2fNp1LgeANWqVSEsrAB79uzzOJU7Rox6lQ0bNjNyxDuZYzfc2IhuTzzAHR0eJDX1iIfp3FS8VInM+1c2r8vOjf7f7YiSxTA+fw1GVyhN6cpl2b090ZOMpyunOfHHgf8ZYzYBvzdaRaAa8MhJHxXkoktFMXLCIABCQkOY/dE8Fi9YQu+U/jz3ck9CQ0I4evQYz3d/2eOkwW/ChOE0bFSPqKhINm5aQv/+Q3lv4jTGjBnIsmXzOHb8OJ0f6OF1TGfUu6YW/+x4K6tXr2fxt/4rKV56cTADB71AWMEwPpk1EYC4ZSt4otvzXkYNWg8M78ZF9WoQEVmUgUvGMGvoNC6qV4MK1SuDtezZsZvJvcYCcFHdf9C2ewfS09LJyMhg8nNvcdixL0WNtTb7DYzx4Z8+yfrF5jJrbfqp7OCiUrWz34Gcsp0pe7yOkK+E+kK8jpCvtI++yusI+cq4rdPNqWyX49Up1toM4LszTiQiInlO14mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDgsx//b/ZnakvTb2d7FeSM1frHXEfKVmCrNvY6Qr1ycUdDrCOclnYmLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw0K9DhAMmje7jiFDXiLE5+Oddz9g4KCRXkcKes1uu4fwIkXw+XyEhIQw7Z3hrN+0hX6D3uRw6hHKlS3Na32eIiI8nJ/WbuDF14YDYLF0vb8TNzZu4PErCE6xsWUY9dYgSpeOxlrLxHenMnb0RMZPGEa1C6sCULx4UZKSDtG4QRuP0wafomVL0nJoF8JLFcday6r/fMkP78yjUPFwWo96hOLlS5G0Yzezu77J0aTDhBUtzE1vPETRclH4QkOIGzuX1dO/8vpl5Iqx1p7VHYSGxZ7dHZwhn8/HujWLadHqn+zY8RvfLZnLnXd1Zd26TV5H+4vU+MVeR8jU7LZ7mDp+OJElimeOdfjXY/R85N/UubImH82Zx874XTza+W5SjxyhQGgBQkND2L1nH7fd05UFM98nNDTEw1cAMVWae7r/vxMTU4qYMqVYtXItERHhLFj8MXfd0ZUNG37O3KbfgGc4mJTMoNdGeJj0r3pFXu11BMJLlyC8dAkSV2+lQHgh7vq0HzMfGEqNdo04ciCFpaNmU7frzRQqXoSvXpnK1Q+3oWCxwnz1ylQKlyzK/QsHMbrWw2QcT/f6pdBz+2RzKtvlejrFGPNe7uMEr7p1rmTz5q388st2jh8/zrRpM2lzc/D9crtg2687qX3FZQBcU+cqPl/0NQCFCxXKLOyjx46BOaWfzfPSrl27WbVyLQDJySls3LCZsuViTtjmlltb8eGM2V7EC3opiQdIXL0VgOMpR9j3czwRZUpSrWkt1szwnwStmbGYas1qA/5PhmHhhQEICy/EkQMpZKRleJL9dGU7nWKMmfXnIeB6Y0wJAGut85/nysWW4dcd8ZnLO3b+Rt06V3qYyA3GGDo/8RzGGNq1bUm7tq24oEolFixewg2N6jP/y8Uk7NqTuf2qNet5fsBQ4ncl8srzPT0/C3dBhYqx1KxZneVxKzPHrmlQh8TEPWzZvM3DZG4oVj6a0jUq8duPmykSXYyUxAOAv+iLRBcD4McJn3Pr+O50iRtBWHgh5jw8As7y7ERey2lOvDywFngbsPhLvDYwOLsHGWM6A50BTEhxfL7wM08qQeW90a8TUyqavfsP8MDjvahSqQL9ej3BK0NHM3bCB1x3bT0KFPjjx6tmjUuY+f5YNm/dznP9B9OwXh0KFgzz8BUEt/DwIkycPIJez7zMoUPJmeO33d6aj2bM8TCZGwoUKUibsd34su9kjiWnnnS7Ko0vI3HtNqbdMYASlWJo9/7T7Fi6IdvHBJucplNqA8uB54Aka+1CINVau8hau+hkD7LWvmWtrW2trR3sBR6/M4EK5ctlLpePLUt8fIKHidwQUyoagKjIEtzQqD4/rd1A1UoVGDdsANPeeZNWNzamQmzZvzzugsoVKVK4MJu2bD3Hid0RGhrKxMkjmDFtFnNmzc8cDwkJoXWbZnz84VwP0wU/X2gIbcZ2Y93H37Lpv3EAHN5zkPDSJQD/vPnhPQcBuLRd48xtDmzbRdKvuyl5wV9/boNZtiVurc2w1g4F7gOeM8aMIJ9d0bIsbgXVqlWhcuUKFChQgPbt2zJ7zvycH3geO5x6hJSUw5n3v136AxdWrcze/f6PqxkZGYydOIX2t7QCYEd8Amlp/i+K4hN28cu2X4ktG/P3Ty4MHzmAjRs2M2rEuyeMX3d9fTZt3KKTjBw0H/Rv9v0cz/K3P8sc2/z5D9S4vSEANW5vyM+fLwfgYPweKjWoAUCR6GJEXlCWpO2J5z70GTilQrbW7gDaGWNuAg6e3UjnVnp6Ot0e783cT/9DiM/HhIlTWbt2o9exgtreffvp1qsfAOlp6bRqdh3X1qvNpGmfMOUj/0f9GxvX59abmgHww6o1jJ80jdDQUHw+Q++eD59wVYv84epranFHx1tZs3o9i77xfyXVr+9gvpi/iFtvb82H0zWVkp3YOhdR47aG7F63nbs/exmAxQOn8f2o2dw8+lEu69CYgzv3MPuhNwFYMvwTWg5+kHvmv4Ix8NUrU0ndn5zdLoLOeX+JoUuC6RLD/CAYLzF0WTBcYpifnLVLDEVEJHioxEVEHKYSFxFxmEpcRMRhKnEREYepxEVEHKYSFxFxmEpcRMRhKnEREYepxEVEHKYSFxFxmEpcRMRhKnEREYepxEVEHKYSFxFxmEpcRMRhKnEREYepxEVEHKYSFxFxmEpcRMRhKnEREYepxEVEHBbqdQA5dSUqNvE6Qr6yb9l4ryPkKz1uesvrCOclnYmLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4rBQrwMEg+bNrmPIkJcI8fl4590PGDhopNeRnDF6zEBatmjC7t17qVOnOQC9nnuc++67gz179gHwYp+BzJu30MOUwa/Fg70pUrgQIT4fISE+pgx6hlFT5vDRF98QWawoAI91akPDWpcCsHHrDl4a8wEpqUcwxvDBwKcpGFbAy5cQNDoN7MKlTa7i0N6DDGjeM3O88T0taHh3M2x6BqsX/MjMV9/PXBdZLorenw9h7rDp/G/cHC9in7bzvsR9Ph/D33iZFq3+yY4dv/HdkrnMnjOfdes2eR3NCZMnzWDsmImMGzfkhPERb47njTfGeZTKTeNfepzIYhEnjN3Zugn33tL0hLG09HSefWMCAx67l4urlOfAoWRCQ0LOZdSg9t2MRSyaOI+7hzycOXbhNTW4rGltXm35FGnH0oiIKnbCY/6v992sWbjiXEfNE+f9dErdOleyefNWfvllO8ePH2fatJm0ubm517Gc8c03S9m3L8nrGOeVJSvWcVGlWC6uUh6AEkUjCAk573+VM21euo7DScknjDXs1JTPR88k7VgaAMl7D2auq9msNnt/TSRh06/nNGdeOe//y5eLLcOvO+Izl3fs/I1y5cp4mCh/eLDLPXz//WeMHjOQEiWK5fyA850xPNj3TTr0fIUZ87/OHJ7y2SJue6I/L4yYxMHkwwBsjU/EGEOXl96kfY9XeOfj+V6ldkbpqmW5oO4l9PykP92m9qFizQsACCtSkKZd2jL3jRkeJzx9Z6XEjTGdjTFxxpi4jIyUs7ELCWJvj5vMpTUaUa9eKxISEnnl1d5eRwp6E1/uwbTBzzKq9yNM+WwRcWs20aFFIz4d9RLTB/ciOrIYr0/4EID09HR+WLeZVx6/j4kDerDg+5V8t2q9x68guPlCQggvHsHrt/TmkwGTuX/k4wDc9Hg7Foz/lGOHj3qc8PRlW+LGmOLGmFeNMeuNMfuMMXuNMesCYyVO9jhr7VvW2trW2to+X3jep85D8TsTqFC+XOZy+diyxMcneJjIfYmJe8jIyMBay7vvTKF2rcu9jhT0YqL8v05RJYrS5OrLWb1pK1ElihES4sPn83Fb02v5adNW/7bRkdSqXo3IYhEULhhGw6tqsG6Lm1MB58qBhL2smLcUgG0rN2MzMogoWZRKV1Tjlmc70ffrN7nu/lY0e/hWGt3t1nRqTmfi04D9wHXW2pLW2ijg+sDYtLMd7lxYFreCatWqULlyBQoUKED79m2ZPUcfT89EmTKlMu+3adOcNWs3epgm+B0+cpSU1COZ95esXEe1iuXYneW7hgXfr+DCiv6TjQZXVGfTtnhSjx4jLT2duLWbuKC8pgCzs2r+Mi6qVx2A0lXKEloglOR9hxjW/kX6XPsofa59lIXvzGX+yI/56r15HqfNnZyuTqlsrX0t64C1NgF4zRhz/9mLde6kp6fT7fHezP30P4T4fEyYOJW1Kp1TNmHCcBo2qkdUVCQbNy2hf/+hNGpYj5o1q2OtZdv2HTz2aC+vYwa1fQcO8fhrYwFIz8igZcPaXHtVDXq9MYH1v+zAGChXKooXunQEoFhEEe5u04SOT/l/NRvWqkGj2pd5lj/Y3Dv8MS6sV52IyKL0WzKKuUOns2Tal3Qa+BC95r1O+vE0JvUY5XXMPGOstSdfacx84AtgorV2V2AsBrgXaGqtvTGnHYSGxZ58B5IrBUN1HXBe2rdsvNcR8pUeN73ldYR8ZcTWqeZUtstpOqUDEAUsCsyJ7wMWAiWBdmeUUEREzli20ynW2v3A04HbCYwx9wHvnqVcIiJyCs7kEsO+eZZCREROS7Zn4saYVSdbBcTkfRwREcmNnK5OiQGa47+kMCsDfHtWEomIyCnLqcTnABHW2r/8yzDGmIVnJZGIiJyynL7Y/Fc26zrmfRwREcmN8/4fwBIRcZlKXETEYSpxERGHqcRFRBymEhcRcZhKXETEYSpxERGHqcRFRBymEhcRcZhKXETEYSpxERGHqcRFRBymEhcRcZhKXETEYSpxERGHGWvtWd1BaFjs2d3BeaR7uUZeR8hXttlUryPkK5O+1v92Ny+FVbjcnMp2OhMXEXGYSlxExGEqcRERh6nERUQcphIXEXGYSlxExGEqcRERh6nERUQcphIXEXGYSlxExGEqcRERh6nERUQcphIXEXGYSlxExGEqcRERh6nERUQcphIXEXGYSlxExGEqcRERh6nERUQcphIXEXGYSlxExGGhXgfwWvny5ZjwzhuUjonGWsvbb7/PmyPGex0rqLUb+CD/aHIlyXsPMqT5UwB0GvEYpaqWBaBQsXCOHExhWKtn8YWGcPtrnYmtURlfaAg/fLSYL0fN9DJ+0Oky6BGualKbg3uT6NmsGwDte3SkdtO62AxL0t4kRvd4g/2J+6ndtC7te3TEZljS09OZ2Hc8G+LWefwKgkvzTg9TpHAhQkJ8hISEMHXUq/TsN5StO+IBOJR8mKIRRZgxdhDHj6fRd9hbrNmwGZ/PxzNd76XOFTU8fgW5c96XeFpaGk8+1ZcfV6wmIiKcpd//ly/+9xXr1m3yOlrQipuxiG8nzqPDkK6ZY+8/Mjzzfuvn7uTIocMA1Gx1NaFhoQxt8TQFCoXR44vXWTHrG/bv2HPOcwerRdMXMG/iXB4e0i1zbPbYj5k2+D8AtLj3Jm7r1oG3nxvDT9+sIu7zpQBUvKQSj498ku43POJJ7mD2zuA+RBYvlrn8+vNPZN4fNOY9IsKLADBj7hcAfPz2YPbuT+KhXgOYMvIVfD53JincSXqWJCQk8uOK1QAkJ6ewfv0mYsuV8ThVcPtl6XoOJyWfdH3Nm+qxYta3mcthhQviC/FRoFAY6cfSOHIo9VzEdMa6pWtJPnDi8UxN/uMYFSpSCGstAEcPH8kcL1ikEGDPScb8wlrLvEVLaHV9AwA2b9vB1VdcCkBUZHGKRYSzZuMWLyPm2nl/Jp5VpUrlueLyS/l+6Y9eR3FWlbqXkLwniT1bEwBYNfd7qjetRe+lowkrHMbsfpNITUrxOKUbOjzZiUb/dz2ph1Loe8fzmeN1ml/NP5+6i+LRxXn1vv4eJgxOxsCDT78MBtrd1JR2rW/MXLf8p3VERRanUnn/1N/FVSvz5ZI4WjZpQELiXtZu3EJC4h4uu6SaV/FzLdclbowpba1NPBthvBQeXoRpU8fRvWcfDh06+VmmZO+KNvVPOAuvcPkF2PQM+l/dlcLFw+k6rQ+bvl7Nvl/z3Y9Qnps66H2mDnqfW7reRot7WjF96BQAls37nmXzvucfdavToUdH+nfq43HS4DJxWD9iokuyd38SnZ/uT5WK5ahdszoAny34JvMsHODWltezZfsO7uj6DGVLl+LyGhfjC3FrgiLbtMaYkn+6RQFLjTGRxpiS2TyuszEmzhgTl5ER/GddoaGhTJ86jg8++JhPPvnM6zjO8oX4uLR5XVbOWZI5dmXbBmxYtJKMtHRS9h5k6/KNlK9Z1cOU7ln8ySKubnnNX8bXLV1L6YoxFI0s6kGq4BUT7a+mqMji3NCgDqvX/wxAWno6X3y9lObX1c/cNjQkhKe73suMsYN4s99THEpOoXL5cp7kPl05veXsAZZnucUBscAPgft/y1r7lrW2trW2ts8XnldZz5pxbw1m3fqfGfbGW15HcVq1ay9j95Z4khL2ZY4diN/DBfX93/YXKFyQildWI3FzvFcRnVGmctnM+3WaXc3OzTsBiKn0x/c1VS6tSoGwAhzaf+ic5wtWh1OPkHI4NfP+t8tXUa1yRQC+W/4TVSqWo0ypqMztU48c5XCq/3uGb5evIiQkhAsqlT/3wc9ATtMpTwJNgSettT8BGGN+sdZWOevJzpEG9etw1523s+qntcQtmw/A88+/ymf/XeBxsuDVcfijVK33D8Iji9JryQg+HzqDZdMWcsXN15wwlQLw7XvzaT+oC93nD8IYiJu+iIT12z1KHpweG96d6tdcStHIYoz67m2mD53CldfXolzVcmRkWPbs3M24XqMBuLrlNTS67XrSj6dz7OhRhj38usfpg8ve/Uk8/qL/mKSnp9OqybVcW/cKAD5beOJUCsC+A0l0eeZljM9H6aiSvPKMe1f6mN+/9T7pBsaUB4YCvwJ9gJXW2lP+PBwaFquvz/NI93KNvI6Qr2yzukomL036uq/XEfKVsAqXm1PZLscZfGvtDmttO2Ah8DlQ5MyiiYhIXjnlr2GttbOA64EbAYwx952tUCIicmpydS2NtTbVWrs6sKjPTiIiHsv2i01jzKqTrQJi8j6OiIjkRk5Xp8QAzYH9fxo3wLd/3VxERM6lnEp8DhBhrV3x5xXGmIVnJZGIiJyybEvcWvuvbNZ1zPs4IiKSG279IwEiInIClbiIiMNU4iIiDlOJi4g4TCUuIuIwlbiIiMNU4iIiDlOJi4g4TCUuIuIwlbiIiMNU4iIiDlOJi4g4TCUuIuIwlbiIiMNU4iIiDlOJi4g4zFhrvc4QFIwxna21b3mdI7/Q8cw7OpZ5K78dT52J/6Gz1wHyGR3PvKMOjgMbAAACOElEQVRjmbfy1fFUiYuIOEwlLiLiMJX4H/LNHFmQ0PHMOzqWeStfHU99sSki4jCdiYuIOEwlLiLiMJU4YIxpYYzZYIz52RjzjNd5XGeMCTHG/GiMmeN1FtcYY94xxiQaY1ZnGStpjPncGLMp8Geklxld8nfHM8u6HsYYa4yJ9iJbXjnvS9wYEwKMBFoC1YF/GmOqe5vKed2AdV6HcNQEoMWfxp4B/metvRD4X2BZTs0E/no8McZUAJoB2891oLx23pc4UBf42Vq7xVp7DJgCtPU4k7OMMeWBm4C3vc7iImvtV8C+Pw23BSYG7k8EbjmnoRx2kuMJMBR4CnD+yg6VOMQCv2ZZ3hEYk9MzDP8vR4bXQfKRGGvtb4H7CUCMl2FcZ4xpC+y01q70OkteUIlLnjHGtAYSrbXLvc6SX1n/NcHOnz16xRhTBOgFvOB1lryiEoedQIUsy+UDY5J7DYA2xpit+KelmhhjJnsbKV/YZYwpCxD4M9HjPC67AKgCrAz8nJYHfjDGlPE01RlQicMy4EJjTBVjTBhwBzDL40xOstY+a60tb62tjP84LrDW3ulxrPxgFnBP4P49wEwPszjNWvuTtba0tbZy4Od0B3CVtTbB42in7bwvcWttGvAIMA//FRXTrLVrvE0l5ytjzAfAEuBiY8wOY8y/gFeBpsaYTcCNgWU5BSc5nvmK/tq9iIjDzvszcRERl6nERUQcphIXEXGYSlxExGEqcRERh6nERUQcphIXEXHY/wNyP9u5CfZxtQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "print_result(nb, test_text, X_test, y_test, cv, True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tuned hpyerparameters :(best parameters)  {'n_neighbors': 100}\n",
      "accuracy : 0.6934279992718004\n"
     ]
    }
   ],
   "source": [
    "param = [\n",
    "    {'n_neighbors': [50,100, 150]},\n",
    "]\n",
    "model = KNeighborsClassifier()\n",
    "cv = GridSearchCV(model, param, cv = 3)\n",
    "kn = Pipeline([\n",
    "                    ('tfidf', vec_model),\n",
    "                    ('clf', cv),\n",
    "               ])\n",
    "kn.fit(X_train, y_train)\n",
    "print(\"tuned hpyerparameters :(best parameters) \", cv.best_params_)\n",
    "print(\"accuracy :\", cv.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted service ID:  [0]\n",
      "Predic_proba :  [[0.34 0.25 0.23 0.18]]\n",
      "Precision:   0.75\n",
      "Recall:   0.59\n",
      "F1-measure:   0.61\n",
      "Accuracy:   0.66\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.97      0.23      0.37       251\n",
      "           4       0.74      0.72      0.73       828\n",
      "          10       0.75      0.76      0.75       735\n",
      "          14       0.52      0.65      0.58       892\n",
      "\n",
      "   micro avg       0.66      0.66      0.66      2706\n",
      "   macro avg       0.75      0.59      0.61      2706\n",
      "weighted avg       0.69      0.66      0.65      2706\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXEAAAEICAYAAACpqsStAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3XmcTfXjx/HX584Yy4xlzDAYe1q+lBZLIpSyJupX6Ev795ukRaFNSiIVWZI1KaJvthZL+lJfkUoxCtlFiGmMdZgxlpn5/P64t2lUZgzDuZ/xfj4e9+Gezzn3nvc9zbzvuZ97yFhrERERN/m8DiAiIqdPJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuAQVY0xhY8xsY0ySMWb6GTxPJ2PM/LzM5hVjTENjzAavc0hwMrpOXE6HMaYj0B24BDgErABettZ+fYbPexfwKFDfWpt2xkGDnDHGAhdaa3/2Oou4SWfikmvGmO7AMGAAEANUBEYBbfPg6SsBG8+HAj8VxphQrzNIkLPW6qbbKd+A4kAy0C6bbQriL/n4wG0YUDCw7jpgB9ADSAR+A+4LrOsLHAOOB/bxL+BFYHKW564MWCA0sHwvsAX/p4FfgE5Zxr/O8rj6wDIgKfBn/SzrFgL9gG8CzzMfiD7Ja/s9/1NZ8t8CtAI2AvuAXlm2rwssAQ4Eth0BhAXWfRV4LSmB19shy/M/DSQAk34fCzzmgsA+rgoslwN2A9d5/bOhmzc3nYlLbl0DFAI+zmab54B6wBXA5fiLrHeW9WXwvxnE4i/qkcaYSGttH/xn91OttRHW2vHZBTHGhAPDgZbW2qL4i3rF32xXEvg0sG0UMAT41BgTlWWzjsB9QGkgDOiZza7L4D8GscALwDjgTqAW0BB43hhTJbBtOvAEEI3/2N0AdAWw1jYKbHN54PVOzfL8JfF/KumcdcfW2s34C36yMaYI8C4w0Vq7MJu8ko+pxCW3ooA9Nvvpjk7AS9baRGvtbvxn2HdlWX88sP64tXYu/rPQi08zTwZwqTGmsLX2N2vtmr/Z5iZgk7V2krU2zVr7AbAeuDnLNu9aazdaa1OBafjfgE7mOP75/+PAFPwF/Ya19lBg/2vxv3lhrV1urf0usN+twFig8Sm8pj7W2qOBPCew1o4Dfga+B8rif9OU85RKXHJrLxCdw1xtOWBbluVtgbHM5/jTm8BhICK3Qay1KfinILoAvxljPjXGXHIKeX7PFJtlOSEXefZaa9MD938v2V1Z1qf+/nhjzEXGmDnGmARjzEH8nzSis3lugN3W2iM5bDMOuBR401p7NIdtJR9TiUtuLQGO4p8HPpl4/FMBv6sYGDsdKUCRLMtlsq601s6z1jbFf0a6Hn+55ZTn90w7TzNTbozGn+tCa20xoBdgcnhMtpeMGWMi8H/PMB54MTBdJOcplbjkirU2Cf888EhjzC3GmCLGmALGmJbGmIGBzT4AehtjShljogPbTz7NXa4AGhljKhpjigPP/r7CGBNjjGkbmBs/in9aJuNvnmMucJExpqMxJtQY0wGoDsw5zUy5URQ4CCQHPiU89Kf1u4CquXzON4A4a+2/8c/1jznjlOIslbjkmrV2MP5rxHvjvzLiV+AR4JPAJv2BOGAV8BPwQ2DsdPb1OTA18FzLObF4fYEc8fiv2GjMX0sSa+1eoDX+K2L24r+ypLW1ds/pZMqlnvi/ND2E/1PC1D+tfxGYaIw5YIxpn9OTGWPaAi3443V2B64yxnTKs8TiFP1lHxERh+lMXETEYSpxERGHqcRFRBymEhcRcdhZ/8d1LipVW9+c5pGdKefiYorzR6gvxOsI+Ur76Ku8jpCvjNs6Pae/TwDoTFxExGkqcRERh6nERUQcphIXEXGYSlxExGEqcRERh6nERUQcphIXEXGYSlxExGEqcRERh6nERUQcphIXEXGYSlxExGEqcRERh6nERUQcphIXEXGYSlxExGEqcRERh6nERUQcphIXEXGYSlxExGEqcRERh4V6HcArC5bPIiX5MBkZ6aSlpXNb07v5x6UX0XfQsxQsFEZaWjp9n3qNVT+u8TpqUBs9ZiAtWzRh9+691KnTPHO8S5d76Pzg3aSnpzPvvwvo3ftVD1O6Iza2LGPGvU7p0lFYa5nw7lTGjJpAv/7P0KJVE44dO84vv2zn4S5PkZR0yOu4QemegQ9Rs0ktDu1N4sXmPQC4+fF2NLzjRpL3HQTgo4H/YfXCH4kqX4qXvhjGri3xAGz5cSOTnxvnWfbTcd6WOMDdtz7I/n1JmctPvvAYI14fx1f/+5bGNzbgyT6PcdctD3qYMPhNnjSDsWMmMm7ckMyxRo2uoXXrptS7uiXHjh2jVKkoDxO6JS0tjd7PDmDlyjVERISzaPFMvlzwNV8u+JoX+wwiPT2dvi89RfceD9HnhYFexw1K385YyJcT/8v9Qx45YfyL8XOYP272X7bfvS2Bl1o9ea7i5TlNp2RhsUQUDQcgomgEiQm7PU4U/L75Zin7srwRAvz7gU4MHjyaY8eOAbB7914vojlp167drFzp//SXnJzChg0/U65sDAsWfE16ejoAy5atoFxsGS9jBrVNS9eRkpTsdYxzJsczcWPMJUBbIDYwtBOYZa1ddzaDnW3WWt6ZPhJrLVMnfsTUSR8z4LnBjJ82gqdf7IbP56NDq/u9jumkCy+sSv0Gdenz4pMcOXKUXr1e5oflq7yO5ZyKFWOpeXkN4uJWnjB+512389GHn3qUyl3X39OCa/6vMVt/2sz0/u9x+GAKANEVSvP8pwNJTU5l5usfsGnZeo+T5k62JW6MeRr4JzAFWBoYLg98YIyZYq11dqKzY+t/sythNyWjI5kwfSSbf95Ki5tvYMDzQ5g/ZwEt297IgGHPc+/tD3sd1TmhISFERhbnusa3UKv25UyaNJIa1Rt6Hcsp4eFFmPT+KJ59uh+HDv1xVtnzya6kpaczbepMD9O5Z+Hk+cwZ/iFYS9sed9Cu991MfGo0SYn7ebr+Q6QcSKbipVV5+K0n6dOsO0eSU72OfMpymk75F1DHWvuqtXZy4PYqUDew7m8ZYzobY+KMMXFJR4JzSmJXYKpk3579fD53ITWvrMGtHVozf84CAD6b+QU1r6rhZURn7YxPYNbMeQAsj1tJRkYG0dElPU7ljtDQUCa9P5JpU2cye9b8zPGOnW6jeYvreeD+JzxM56ZDe5KwGRlYa1k85QuqXF4NgLRjaaQc8L9Jbl+9hd3bdxFTpayXUXMtpxLPAMr9zXjZwLq/Za19y1pb21pbu3ihUmeS76woXKQQ4eFFMu83uO5qNq3fTGLCburWrwXANQ3rsHXLr17GdNbs2fNp1LgeANWqVSEsrAB79uzzOJU7Rox6lQ0bNjNyxDuZYzfc2IhuTzzAHR0eJDX1iIfp3FS8VInM+1c2r8vOjf7f7YiSxTA+fw1GVyhN6cpl2b090ZOMpyunOfHHgf8ZYzYBvzdaRaAa8MhJHxXkoktFMXLCIABCQkOY/dE8Fi9YQu+U/jz3ck9CQ0I4evQYz3d/2eOkwW/ChOE0bFSPqKhINm5aQv/+Q3lv4jTGjBnIsmXzOHb8OJ0f6OF1TGfUu6YW/+x4K6tXr2fxt/4rKV56cTADB71AWMEwPpk1EYC4ZSt4otvzXkYNWg8M78ZF9WoQEVmUgUvGMGvoNC6qV4MK1SuDtezZsZvJvcYCcFHdf9C2ewfS09LJyMhg8nNvcdixL0WNtTb7DYzx4Z8+yfrF5jJrbfqp7OCiUrWz34Gcsp0pe7yOkK+E+kK8jpCvtI++yusI+cq4rdPNqWyX49Up1toM4LszTiQiInlO14mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDgsx//b/ZnakvTb2d7FeSM1frHXEfKVmCrNvY6Qr1ycUdDrCOclnYmLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw0K9DhAMmje7jiFDXiLE5+Oddz9g4KCRXkcKes1uu4fwIkXw+XyEhIQw7Z3hrN+0hX6D3uRw6hHKlS3Na32eIiI8nJ/WbuDF14YDYLF0vb8TNzZu4PErCE6xsWUY9dYgSpeOxlrLxHenMnb0RMZPGEa1C6sCULx4UZKSDtG4QRuP0wafomVL0nJoF8JLFcday6r/fMkP78yjUPFwWo96hOLlS5G0Yzezu77J0aTDhBUtzE1vPETRclH4QkOIGzuX1dO/8vpl5Iqx1p7VHYSGxZ7dHZwhn8/HujWLadHqn+zY8RvfLZnLnXd1Zd26TV5H+4vU+MVeR8jU7LZ7mDp+OJElimeOdfjXY/R85N/UubImH82Zx874XTza+W5SjxyhQGgBQkND2L1nH7fd05UFM98nNDTEw1cAMVWae7r/vxMTU4qYMqVYtXItERHhLFj8MXfd0ZUNG37O3KbfgGc4mJTMoNdGeJj0r3pFXu11BMJLlyC8dAkSV2+lQHgh7vq0HzMfGEqNdo04ciCFpaNmU7frzRQqXoSvXpnK1Q+3oWCxwnz1ylQKlyzK/QsHMbrWw2QcT/f6pdBz+2RzKtvlejrFGPNe7uMEr7p1rmTz5q388st2jh8/zrRpM2lzc/D9crtg2687qX3FZQBcU+cqPl/0NQCFCxXKLOyjx46BOaWfzfPSrl27WbVyLQDJySls3LCZsuViTtjmlltb8eGM2V7EC3opiQdIXL0VgOMpR9j3czwRZUpSrWkt1szwnwStmbGYas1qA/5PhmHhhQEICy/EkQMpZKRleJL9dGU7nWKMmfXnIeB6Y0wJAGut85/nysWW4dcd8ZnLO3b+Rt06V3qYyA3GGDo/8RzGGNq1bUm7tq24oEolFixewg2N6jP/y8Uk7NqTuf2qNet5fsBQ4ncl8srzPT0/C3dBhYqx1KxZneVxKzPHrmlQh8TEPWzZvM3DZG4oVj6a0jUq8duPmykSXYyUxAOAv+iLRBcD4McJn3Pr+O50iRtBWHgh5jw8As7y7ERey2lOvDywFngbsPhLvDYwOLsHGWM6A50BTEhxfL7wM08qQeW90a8TUyqavfsP8MDjvahSqQL9ej3BK0NHM3bCB1x3bT0KFPjjx6tmjUuY+f5YNm/dznP9B9OwXh0KFgzz8BUEt/DwIkycPIJez7zMoUPJmeO33d6aj2bM8TCZGwoUKUibsd34su9kjiWnnnS7Ko0vI3HtNqbdMYASlWJo9/7T7Fi6IdvHBJucplNqA8uB54Aka+1CINVau8hau+hkD7LWvmWtrW2trR3sBR6/M4EK5ctlLpePLUt8fIKHidwQUyoagKjIEtzQqD4/rd1A1UoVGDdsANPeeZNWNzamQmzZvzzugsoVKVK4MJu2bD3Hid0RGhrKxMkjmDFtFnNmzc8cDwkJoXWbZnz84VwP0wU/X2gIbcZ2Y93H37Lpv3EAHN5zkPDSJQD/vPnhPQcBuLRd48xtDmzbRdKvuyl5wV9/boNZtiVurc2w1g4F7gOeM8aMIJ9d0bIsbgXVqlWhcuUKFChQgPbt2zJ7zvycH3geO5x6hJSUw5n3v136AxdWrcze/f6PqxkZGYydOIX2t7QCYEd8Amlp/i+K4hN28cu2X4ktG/P3Ty4MHzmAjRs2M2rEuyeMX3d9fTZt3KKTjBw0H/Rv9v0cz/K3P8sc2/z5D9S4vSEANW5vyM+fLwfgYPweKjWoAUCR6GJEXlCWpO2J5z70GTilQrbW7gDaGWNuAg6e3UjnVnp6Ot0e783cT/9DiM/HhIlTWbt2o9exgtreffvp1qsfAOlp6bRqdh3X1qvNpGmfMOUj/0f9GxvX59abmgHww6o1jJ80jdDQUHw+Q++eD59wVYv84epranFHx1tZs3o9i77xfyXVr+9gvpi/iFtvb82H0zWVkp3YOhdR47aG7F63nbs/exmAxQOn8f2o2dw8+lEu69CYgzv3MPuhNwFYMvwTWg5+kHvmv4Ix8NUrU0ndn5zdLoLOeX+JoUuC6RLD/CAYLzF0WTBcYpifnLVLDEVEJHioxEVEHKYSFxFxmEpcRMRhKnEREYepxEVEHKYSFxFxmEpcRMRhKnEREYepxEVEHKYSFxFxmEpcRMRhKnEREYepxEVEHKYSFxFxmEpcRMRhKnEREYepxEVEHKYSFxFxmEpcRMRhKnEREYepxEVEHBbqdQA5dSUqNvE6Qr6yb9l4ryPkKz1uesvrCOclnYmLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4jCVuIiIw1TiIiIOU4mLiDhMJS4i4rBQrwMEg+bNrmPIkJcI8fl4590PGDhopNeRnDF6zEBatmjC7t17qVOnOQC9nnuc++67gz179gHwYp+BzJu30MOUwa/Fg70pUrgQIT4fISE+pgx6hlFT5vDRF98QWawoAI91akPDWpcCsHHrDl4a8wEpqUcwxvDBwKcpGFbAy5cQNDoN7MKlTa7i0N6DDGjeM3O88T0taHh3M2x6BqsX/MjMV9/PXBdZLorenw9h7rDp/G/cHC9in7bzvsR9Ph/D33iZFq3+yY4dv/HdkrnMnjOfdes2eR3NCZMnzWDsmImMGzfkhPERb47njTfGeZTKTeNfepzIYhEnjN3Zugn33tL0hLG09HSefWMCAx67l4urlOfAoWRCQ0LOZdSg9t2MRSyaOI+7hzycOXbhNTW4rGltXm35FGnH0oiIKnbCY/6v992sWbjiXEfNE+f9dErdOleyefNWfvllO8ePH2fatJm0ubm517Gc8c03S9m3L8nrGOeVJSvWcVGlWC6uUh6AEkUjCAk573+VM21euo7DScknjDXs1JTPR88k7VgaAMl7D2auq9msNnt/TSRh06/nNGdeOe//y5eLLcOvO+Izl3fs/I1y5cp4mCh/eLDLPXz//WeMHjOQEiWK5fyA850xPNj3TTr0fIUZ87/OHJ7y2SJue6I/L4yYxMHkwwBsjU/EGEOXl96kfY9XeOfj+V6ldkbpqmW5oO4l9PykP92m9qFizQsACCtSkKZd2jL3jRkeJzx9Z6XEjTGdjTFxxpi4jIyUs7ELCWJvj5vMpTUaUa9eKxISEnnl1d5eRwp6E1/uwbTBzzKq9yNM+WwRcWs20aFFIz4d9RLTB/ciOrIYr0/4EID09HR+WLeZVx6/j4kDerDg+5V8t2q9x68guPlCQggvHsHrt/TmkwGTuX/k4wDc9Hg7Foz/lGOHj3qc8PRlW+LGmOLGmFeNMeuNMfuMMXuNMesCYyVO9jhr7VvW2trW2to+X3jep85D8TsTqFC+XOZy+diyxMcneJjIfYmJe8jIyMBay7vvTKF2rcu9jhT0YqL8v05RJYrS5OrLWb1pK1ElihES4sPn83Fb02v5adNW/7bRkdSqXo3IYhEULhhGw6tqsG6Lm1MB58qBhL2smLcUgG0rN2MzMogoWZRKV1Tjlmc70ffrN7nu/lY0e/hWGt3t1nRqTmfi04D9wHXW2pLW2ijg+sDYtLMd7lxYFreCatWqULlyBQoUKED79m2ZPUcfT89EmTKlMu+3adOcNWs3epgm+B0+cpSU1COZ95esXEe1iuXYneW7hgXfr+DCiv6TjQZXVGfTtnhSjx4jLT2duLWbuKC8pgCzs2r+Mi6qVx2A0lXKEloglOR9hxjW/kX6XPsofa59lIXvzGX+yI/56r15HqfNnZyuTqlsrX0t64C1NgF4zRhz/9mLde6kp6fT7fHezP30P4T4fEyYOJW1Kp1TNmHCcBo2qkdUVCQbNy2hf/+hNGpYj5o1q2OtZdv2HTz2aC+vYwa1fQcO8fhrYwFIz8igZcPaXHtVDXq9MYH1v+zAGChXKooXunQEoFhEEe5u04SOT/l/NRvWqkGj2pd5lj/Y3Dv8MS6sV52IyKL0WzKKuUOns2Tal3Qa+BC95r1O+vE0JvUY5XXMPGOstSdfacx84AtgorV2V2AsBrgXaGqtvTGnHYSGxZ58B5IrBUN1HXBe2rdsvNcR8pUeN73ldYR8ZcTWqeZUtstpOqUDEAUsCsyJ7wMWAiWBdmeUUEREzli20ynW2v3A04HbCYwx9wHvnqVcIiJyCs7kEsO+eZZCREROS7Zn4saYVSdbBcTkfRwREcmNnK5OiQGa47+kMCsDfHtWEomIyCnLqcTnABHW2r/8yzDGmIVnJZGIiJyynL7Y/Fc26zrmfRwREcmN8/4fwBIRcZlKXETEYSpxERGHqcRFRBymEhcRcZhKXETEYSpxERGHqcRFRBymEhcRcZhKXETEYSpxERGHqcRFRBymEhcRcZhKXETEYSpxERGHGWvtWd1BaFjs2d3BeaR7uUZeR8hXttlUryPkK5O+1v92Ny+FVbjcnMp2OhMXEXGYSlxExGEqcRERh6nERUQcphIXEXGYSlxExGEqcRERh6nERUQcphIXEXGYSlxExGEqcRERh6nERUQcphIXEXGYSlxExGEqcRERh6nERUQcphIXEXGYSlxExGEqcRERh6nERUQcphIXEXGYSlxExGGhXgfwWvny5ZjwzhuUjonGWsvbb7/PmyPGex0rqLUb+CD/aHIlyXsPMqT5UwB0GvEYpaqWBaBQsXCOHExhWKtn8YWGcPtrnYmtURlfaAg/fLSYL0fN9DJ+0Oky6BGualKbg3uT6NmsGwDte3SkdtO62AxL0t4kRvd4g/2J+6ndtC7te3TEZljS09OZ2Hc8G+LWefwKgkvzTg9TpHAhQkJ8hISEMHXUq/TsN5StO+IBOJR8mKIRRZgxdhDHj6fRd9hbrNmwGZ/PxzNd76XOFTU8fgW5c96XeFpaGk8+1ZcfV6wmIiKcpd//ly/+9xXr1m3yOlrQipuxiG8nzqPDkK6ZY+8/Mjzzfuvn7uTIocMA1Gx1NaFhoQxt8TQFCoXR44vXWTHrG/bv2HPOcwerRdMXMG/iXB4e0i1zbPbYj5k2+D8AtLj3Jm7r1oG3nxvDT9+sIu7zpQBUvKQSj498ku43POJJ7mD2zuA+RBYvlrn8+vNPZN4fNOY9IsKLADBj7hcAfPz2YPbuT+KhXgOYMvIVfD53JincSXqWJCQk8uOK1QAkJ6ewfv0mYsuV8ThVcPtl6XoOJyWfdH3Nm+qxYta3mcthhQviC/FRoFAY6cfSOHIo9VzEdMa6pWtJPnDi8UxN/uMYFSpSCGstAEcPH8kcL1ikEGDPScb8wlrLvEVLaHV9AwA2b9vB1VdcCkBUZHGKRYSzZuMWLyPm2nl/Jp5VpUrlueLyS/l+6Y9eR3FWlbqXkLwniT1bEwBYNfd7qjetRe+lowkrHMbsfpNITUrxOKUbOjzZiUb/dz2ph1Loe8fzmeN1ml/NP5+6i+LRxXn1vv4eJgxOxsCDT78MBtrd1JR2rW/MXLf8p3VERRanUnn/1N/FVSvz5ZI4WjZpQELiXtZu3EJC4h4uu6SaV/FzLdclbowpba1NPBthvBQeXoRpU8fRvWcfDh06+VmmZO+KNvVPOAuvcPkF2PQM+l/dlcLFw+k6rQ+bvl7Nvl/z3Y9Qnps66H2mDnqfW7reRot7WjF96BQAls37nmXzvucfdavToUdH+nfq43HS4DJxWD9iokuyd38SnZ/uT5WK5ahdszoAny34JvMsHODWltezZfsO7uj6DGVLl+LyGhfjC3FrgiLbtMaYkn+6RQFLjTGRxpiS2TyuszEmzhgTl5ER/GddoaGhTJ86jg8++JhPPvnM6zjO8oX4uLR5XVbOWZI5dmXbBmxYtJKMtHRS9h5k6/KNlK9Z1cOU7ln8ySKubnnNX8bXLV1L6YoxFI0s6kGq4BUT7a+mqMji3NCgDqvX/wxAWno6X3y9lObX1c/cNjQkhKe73suMsYN4s99THEpOoXL5cp7kPl05veXsAZZnucUBscAPgft/y1r7lrW2trW2ts8XnldZz5pxbw1m3fqfGfbGW15HcVq1ay9j95Z4khL2ZY4diN/DBfX93/YXKFyQildWI3FzvFcRnVGmctnM+3WaXc3OzTsBiKn0x/c1VS6tSoGwAhzaf+ic5wtWh1OPkHI4NfP+t8tXUa1yRQC+W/4TVSqWo0ypqMztU48c5XCq/3uGb5evIiQkhAsqlT/3wc9ATtMpTwJNgSettT8BGGN+sdZWOevJzpEG9etw1523s+qntcQtmw/A88+/ymf/XeBxsuDVcfijVK33D8Iji9JryQg+HzqDZdMWcsXN15wwlQLw7XvzaT+oC93nD8IYiJu+iIT12z1KHpweG96d6tdcStHIYoz67m2mD53CldfXolzVcmRkWPbs3M24XqMBuLrlNTS67XrSj6dz7OhRhj38usfpg8ve/Uk8/qL/mKSnp9OqybVcW/cKAD5beOJUCsC+A0l0eeZljM9H6aiSvPKMe1f6mN+/9T7pBsaUB4YCvwJ9gJXW2lP+PBwaFquvz/NI93KNvI6Qr2yzukomL036uq/XEfKVsAqXm1PZLscZfGvtDmttO2Ah8DlQ5MyiiYhIXjnlr2GttbOA64EbAYwx952tUCIicmpydS2NtTbVWrs6sKjPTiIiHsv2i01jzKqTrQJi8j6OiIjkRk5Xp8QAzYH9fxo3wLd/3VxERM6lnEp8DhBhrV3x5xXGmIVnJZGIiJyybEvcWvuvbNZ1zPs4IiKSG279IwEiInIClbiIiMNU4iIiDlOJi4g4TCUuIuIwlbiIiMNU4iIiDlOJi4g4TCUuIuIwlbiIiMNU4iIiDlOJi4g4TCUuIuIwlbiIiMNU4iIiDlOJi4g4zFhrvc4QFIwxna21b3mdI7/Q8cw7OpZ5K78dT52J/6Gz1wHyGR3PvKMOjgMbAAACOElEQVRjmbfy1fFUiYuIOEwlLiLiMJX4H/LNHFmQ0PHMOzqWeStfHU99sSki4jCdiYuIOEwlLiLiMJU4YIxpYYzZYIz52RjzjNd5XGeMCTHG/GiMmeN1FtcYY94xxiQaY1ZnGStpjPncGLMp8Geklxld8nfHM8u6HsYYa4yJ9iJbXjnvS9wYEwKMBFoC1YF/GmOqe5vKed2AdV6HcNQEoMWfxp4B/metvRD4X2BZTs0E/no8McZUAJoB2891oLx23pc4UBf42Vq7xVp7DJgCtPU4k7OMMeWBm4C3vc7iImvtV8C+Pw23BSYG7k8EbjmnoRx2kuMJMBR4CnD+yg6VOMQCv2ZZ3hEYk9MzDP8vR4bXQfKRGGvtb4H7CUCMl2FcZ4xpC+y01q70OkteUIlLnjHGtAYSrbXLvc6SX1n/NcHOnz16xRhTBOgFvOB1lryiEoedQIUsy+UDY5J7DYA2xpit+KelmhhjJnsbKV/YZYwpCxD4M9HjPC67AKgCrAz8nJYHfjDGlPE01RlQicMy4EJjTBVjTBhwBzDL40xOstY+a60tb62tjP84LrDW3ulxrPxgFnBP4P49wEwPszjNWvuTtba0tbZy4Od0B3CVtTbB42in7bwvcWttGvAIMA//FRXTrLVrvE0l5ytjzAfAEuBiY8wOY8y/gFeBpsaYTcCNgWU5BSc5nvmK/tq9iIjDzvszcRERl6nERUQcphIXEXGYSlxExGEqcRERh6nERUQcphIXEXHY/wNyP9u5CfZxtQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "print_result(kn, test_text, X_test, y_test, cv, True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "from mlxtend.classifier import StackingClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/www/pvm/jupiter/lib/python3.6/site-packages/sklearn/svm/base.py:931: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n",
      "/var/www/pvm/jupiter/lib/python3.6/site-packages/sklearn/svm/base.py:931: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n",
      "/var/www/pvm/jupiter/lib/python3.6/site-packages/sklearn/svm/base.py:931: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n",
      "/var/www/pvm/jupiter/lib/python3.6/site-packages/sklearn/svm/base.py:931: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n",
      "/var/www/pvm/jupiter/lib/python3.6/site-packages/sklearn/svm/base.py:931: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n",
      "/var/www/pvm/jupiter/lib/python3.6/site-packages/sklearn/svm/base.py:931: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tuned hpyerparameters :(best parameters)  {'meta_classifier__C': 0.1}\n",
      "accuracy : 0.585654469324595\n"
     ]
    }
   ],
   "source": [
    "clf1 = KNeighborsClassifier(n_neighbors=1)\n",
    "clf2 = RandomForestClassifier(max_depth = 20, n_estimators = 50, random_state=1)\n",
    "clf3 = LinearSVC(loss = 'hinge', penalty = 'l2',max_iter=2000)\n",
    "lr = LogisticRegression(multi_class = 'auto', penalty = 'l2', solver = 'newton-cg')\n",
    "sclf = StackingClassifier(classifiers=[clf1, clf2, clf3], \n",
    "                          meta_classifier=lr)\n",
    "params = {'meta_classifier__C': [0.1, 5, 10.0]}\n",
    "\n",
    "grid = GridSearchCV(estimator=sclf, \n",
    "                    param_grid=params, \n",
    "                    cv=5,\n",
    "                    refit=True)\n",
    "stack = Pipeline([\n",
    "                    ('tfidf', vec_model),\n",
    "                    ('clf', grid),\n",
    "               ])\n",
    "stack.fit(X_train, y_train)\n",
    "print(\"tuned hpyerparameters :(best parameters) \", grid.best_params_)\n",
    "print(\"accuracy :\", grid.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted service ID:  [0]\n",
      "Predic_proba :  [[9.49958137e-01 4.98026447e-02 2.39217876e-04 8.86212713e-10]]\n",
      "Precision:   0.63\n",
      "Recall:   0.58\n",
      "F1-measure:   0.59\n",
      "Accuracy:   0.58\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.80      0.60      0.68       251\n",
      "           4       0.56      0.81      0.66       828\n",
      "          10       0.50      0.56      0.53       735\n",
      "          14       0.66      0.37      0.47       892\n",
      "\n",
      "   micro avg       0.58      0.58      0.58      2706\n",
      "   macro avg       0.63      0.58      0.59      2706\n",
      "weighted avg       0.60      0.58      0.57      2706\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaMAAAEICAYAAADyTpvZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xd4FlX6xvHvk0aRJr2pKIoNEaWoKAjYFUVX1r6uXbGj6CoWQLGBLtgBFVBxEXF/lkXsAoIKSBUQpSgovUgLPcnz+2Mm8SUSSSByErg/18XlvGfOzDwzSd77PWcm0dwdERGRkJJCFyAiIqIwEhGR4BRGIiISnMJIRESCUxiJiEhwCiMREQlOYSTyFzCzUmb2PzNbbWZDdmI/l5rZJ4VZWyhm1tzMfgxdhxRNpt8zkj2ZmV0C3AEcAqwFJgOPuPvondzvP4BbgGbunrHThRZxZubAQe4+O3QtUjxpZCR7LDO7A+gFPApUA/YFXgDaFsLu9wNm7glBlB9mlhK6BinaFEayRzKz8sBDwE3u/n/uvs7dt7j7/9z9rrhPCTPrZWYL43+9zKxEvK6lmc03szvNbKmZLTKzK+N1XYEHgQvNLN3MrjazLmY2MOH4dczMs9+kzewKM/vJzNaa2c9mdmlC++iE7ZqZ2bfx9N+3ZtYsYd0IM3vYzL6K9/OJmVXO4/yz6787of5zzexMM5tpZr+ZWaeE/k3N7BszWxX3fc7M0uJ1X8bdpsTne2HC/v9lZouB/tlt8TZ142McHb+uaWbLzKzlTn1hpdhSGMme6jigJPDOn/S5DzgWaAgcCTQF7k9YXx0oD9QCrgaeN7O93b0z0WhrsLuXcfdX/qwQM9sLeAY4w93LAs2Ipgtz96sIfBD3rQT8G/jAzColdLsEuBKoCqQBHf/k0NWJrkEtovB8CbgMaAQ0Bx4ws/3jvplAB6Ay0bU7CbgRwN1bxH2OjM93cML+KxKNEq9LPLC7zwH+BQw0s9JAf+BVdx/xJ/XKbkxhJHuqSsDy7UyjXQo85O5L3X0Z0BX4R8L6LfH6Le4+DEgHDt7BerKA+mZWyt0Xufv0bfQ5C5jl7q+7e4a7DwJ+AM5O6NPf3We6+wbgLaIgzcsWovtjW4A3iYLmaXdfGx//e6IQxt0nuPuY+LhzgT7Aifk4p87uvimuZyvu/hIwGxgL1CAKf9lDKYxkT7UCqLydexk1gXkJr+fFbTn7yBVm64EyBS3E3dcBFwI3AIvM7AMzOyQf9WTXVCvh9eIC1LPC3TPj5eywWJKwfkP29mZWz8yGmtliM1tDNPLb5hRggmXuvnE7fV4C6gPPuvum7fSV3ZjCSPZU3wCbgHP/pM9CoimmbPvGbTtiHVA64XX1xJXu/rG7n0I0QviB6E16e/Vk17RgB2sqiBeJ6jrI3csBnQDbzjZ/+qiumZUheoDkFaBLPA0peyiFkeyR3H010X2S5+Mb96XNLNXMzjCz7nG3QcD9ZlYlfhDgQWBgXvvcjslACzPbN3544t7sFWZWzczaxveONhFN92VtYx/DgHpmdomZpZjZhcBhwNAdrKkgygJrgPR41NY+1/olwAEF3OfTwHh3v4boXljvna5Sii2Fkeyx3P0pot8xuh9YBvwK3Ay8G3fpBowHvgOmAhPjth051qfA4HhfE9g6QJLiOhYCvxHdi8n9Zo+7rwDaAHcSTTPeDbRx9+U7UlMBdSR6OGIt0ahtcK71XYBX46ftLtjezsysLXA6v5/nHcDR2U8Ryp5Hv/QqIiLBaWQkIiLBKYxERCQ4hZGIiASnMBIRkeD0xwvzqd1+5+hJj0IyYuUPoUsQydO6Lfrd28K0YcO87f0+GqCRkYiIFAEKIxERCU5hJCIiwSmMREQkOIWRiIgEpzASEZHgFEYiIhKcwkhERIJTGImISHAKIxERCU5hJCIiwSmMREQkOIWRiIgEpzASEZHgFEYiIhKcwkhERIJTGImISHAKIxERCU5hJCIiwSmMREQkOIWRiIgEpzASEZHgFEbFzI09buWVCa/x70+ezWm74PaL6TO2Pz2G9aLHsF4c1apRzrrzbmzHsyP78PQXL3Bki6NClFysJCUl8cWod/nPW30A2He/2nz8xRDGTf6Ul/v3IjU1NXCFxUOJEml8MvxtRnz1PqPHfsC/Ot0KQPMWx/LFl+8wasxQnuv9BMnJyYErLR5q167BRx+9ycSJnzFhwqfcdNOVADz6aCcmT/6cceM+YvDgPpQvXy5wpTsueBiZ2TQz+97MJpvZAjPrErqmomz4kM/p9s8uf2j/4JX3uOvM27nrzNuZNHwCALUP2ofjz25Oh1Nu4pF/duXabjeQlBT8S16kXd/+n8yaOSfn9YNdO9L7+QE0bXgKq1at5rLL2wWsrvjYtGkz57W5nJbHn0PL49vS+uTmNGl6FM/1foJrr+pA82PbMP/XBVx0yXmhSy0WMjIyueeebhx99MmceOK5XH/95RxyyEF8/vkoGjU6laZNT2fWrJ+5664bQ5e6w4rKO9MZ7t4Q6Bm6kKJuxrjppK9Kz1ffJqccw1f/G0XG5gyW/rqExXMXcWDDg/7iCouvGjWrccppLRn46pCctuYnHsf7734EwJuD3uGMNieHKq/YWbduPQCpqSmkpqSQmZXJ5i1bmDN7LgAjvviaNm1PC1hh8bF48VImT54GQHr6On74YTY1a1bj889HkZmZCcC4cZOoVatGyDJ3SlEIo1RgU2KDmbU0sy/N7AMz+9HMeptZkpldZWa9Evpda2Y9zayOmW2IR1eTzey1eP1cM6scLw80s2nxcrKZ9TCzb83sOzO7fhee71/i9MvP4qmPnuHGHreyV7m9AKhYvRLLFy3P6bNi8QoqVq8UqsQi75HH76Prg93JysoCoGLFvVm9ek3OD/vCBYupUaNayBKLlaSkJIaPfo8Zc75hxPCvmDj+O1KSk2l4VH0Azj73NGrVqh64yuJn331r07Dh4Xz77eSt2i+//AI+/nhEmKIKQVEIo7LA2m20NwVuAQ4D6gJ/A94Czjaz7In7K4F+8fIcd28Y/7s8cUdmdgRQP6HpamC1uzcBmgDXmtn+hXVCu9rHAz/k5hbX0/GM21i59Df++cDVoUsqdk49vSXLl69gyuTpoUvZbWRlZdHqhLY0OLQFRzdqwCGHHsS1V3Xg4cc68cnwt0lPX0dmZlboMouVvfYqzaBBvbnrrodYu/b3GZK7776ZzMwM3nzznYDV7ZygYWRmyUBZd1+3jdXj3P0nd88EBgEnuHs68AXQxswOAVLdfWo+DtUN6Jzw+lTgcjObDIwFKgF/mL8ys+vMbLyZjf8pfV7BTm4XWr18FVlZWbg7nw36hAOPjE7lt8UrqFyjck6/StUr8dviFaHKLNKaHtOI0884iYlTv6Bv/56c0OJYHu1+H+XLl8u5yV6zVnUWLVoSuNLiZ83qtYweNZaTTm7O+HGTOfv0Szi1VTu++epb5sz+OXR5xUZKSgqDBvVm8OB3ee+9j3LaL7usHWeeeRJXXHFbwOp2XuiR0QHAzDzWeR6vXwauIBoV9c/HMZoB6cCUhDYDbkkYSe3v7p/8oQD3vu7e2N0bH1Bmv3wcKowKVffOWT7mtGP59ccoOL/9dCzHn92clLQUqu5TjRr712T25FmhyizSunV9KvoEf0RrrruyA6O/HMMN13Rk9JdjOOfc0wG46OLz+PCDzwNXWjxUqrQ35cqXBaBkyRKc2Op4Zs36icqVKwKQlpbKrbdfx6v93gxZZrHSu3d3fvxxNs8883JO2ymnnMgdd9xAu3ZXs2HDxoDV7byUwMe/APgmj3VN46mzecCFQF8Adx9rZvsARwMN8nGMLsDfc7V9DLQ3sy/cfYuZ1QMW5DFCK1Juf6Yjhx9Xn7J7l6PPmH4M7jmIw4+tT53D9geHpfOX0KfTCwDMn/UrX38wml6fPU9mRiYvP9A7536I5M9DnZ/kpf49ufeB25k65XveeG3I9jcSqlWvGj+6nURSUhLvvfMhn3w0gi4P382pp7ciKcno/8ogRn05JnSpxUKzZo259NLzmTp1BmPGDAOgc+cePPVUF0qUSGPo0IFA9BDDrbfeF7LUHWbuuQcgu+jAZu2BF4jCZlXcXAVIBq4DOhLdSzoQGA7c6O5Z8bb3AA3d/aL4dR1gqLsn3hfCzOYCH7p7+8Q+ZpZENHV3NtEoaRlwrruvzqvedvudE+ZC7YZGrPwhdAkieVq3ZdP2O0m+bdgwz/LTL+TIqBpwpbsPSGyMf8+oIrDG3dvkse0JJDwG7u5z2foBhez2OtvqE4dap/ifiIgEFvqeUYGYWQUzmwlscHdN3ouI7CZCjoy68ceHFHLac4+YiBpXAfX+4rpERGQXCxZG7p5RkHYREdl9FatpOhER2T0pjEREJDiFkYiIBKcwEhGR4BRGIiISnMJIRESCUxiJiEhwCiMREQlOYSQiIsEpjEREJDiFkYiIBKcwEhGR4BRGIiISnMJIRESCUxiJiEhwCiMREQlOYSQiIsEpjEREJLhg/9vx4uZ/SyaFLmG3kT5/ZOgSdit167UNXcJupXqpiqFL2CNpZCQiIsEpjEREJDiFkYiIBKcwEhGR4BRGIiISnMJIRESCUxiJiEhwCiMREQlOYSQiIsEpjEREJDiFkYiIBKcwEhGR4BRGIiISnMJIRESCUxiJiEhwCiMREQlOYSQiIsEpjEREJDiFkYiIBKcwEhGR4BRGIiISnMJIRESCUxiJiEhwKaELkB3Xp8+TnHnGSSxbtoKjG50MwMDXX6BevQMAKF+hHKtXraHpMaeHLLNIW7M2nc6P92L2T/PAjIc7daBh/UN5Y8h7vPl/Q0lKSqJFs6bcedPVbNmyha7dn2X6D7OwJOOe226g6dENQp9CkfXV5I9Yl76ezMxMMjMyaXPSRRxW/2Ae/fcDlChRgsyMTO67qxtTJk4LXWqx8I/rL+L8S87BcWbNmMP9t3Vj86bNANz7yB2cd3Ebmh7QOnCVO67AYWRm1YFeQBNgFbAEuN3dZxZybbIdr78+hBdfHEC/V3rltF32jxtzlp94/AFWr1kTorRi4/FevTn+mMb0fOR+tmzZwoaNmxg3YQrDR4/hv68+T1paGitWrgLg7fc/AuCd119kxcpVtL/zAd58+WmSkjTBkJcLz7mKlb+tynndqesd9OremxGfjabVyc3p1OUOLjznqoAVFg9Vq1fh0msuoG3zi9m0cRNP9u3GGeeewnuDP+DwIw+hXPmyoUvcaQX6KTIzA94BRrh7XXdvBNwLVPsripM/N3r0WFauXJXn+vPbteGtwe/twoqKl7Xp65gwZRrnn30aAKmpqZQrW4bB737A1ZddQFpaGgCV9q4AwJy5v9C00ZE5bWXL7MX0H2aFKb6YcnfKlt0LgLLlyrBk8bLAFRUfKcnJlChZguTkZEqVLsmyxctISkrizs638NRDz4Uub6cV9CNdK2CLu/fObnD3KcAkM/vczCaa2VQzawtgZnXMLGcMbmbtzGyAmdU1s8nxv8yE5Zrxuo/MbIKZjTKzQ+Jt3zOzy+Pl683sjXh5hJk1TjhGevzflmY2NF6uaGarzKxj4jZmlmxm75vZlTtw7Yq0E044hqVLljN7ztzQpRRZCxYuZu8K5bn/kX/T7oqbePCxXqzfsJG5vyxgwpRpXHzt7Vxx011MnfEjAAcfuD8jRo8hIyOT+QsX8/2Ps1m8RG+meXF3Bv63Dx98MZhL/tkOgK6dnqBT1zsZM/VT7n/oTp54qNd29iIASxcvY8CLb/DZxHcZ/t1Q1q5Zx9cjx3HJ1e0Y/vEoli9dEbrEnVbQabr6wIRttG8EznP3NWZWGRhjZu/ntRN3nwM0hCg83L1h9joz+xy4wd1nmdkxwAtAa+A64Csz+xm4Ezi2AHXfC/yyjfY+wBh377+tjczsuvi4JKdUIDm5TAEOGdaFF7Tlrbc0KvozGZmZzJg5m04d2tPg8EN4rFdvXnn9LTIzM1mzZi3/6duTaTNm0vGBx/hoSH/OO+s0fpr7KxdefSs1q1elYf1DSUrWFF1ezj/znyxZtJRKlSvyxv/1ZfbMnznrnFN46L7ufPi/z2hz7mn0eOYhLvnbtaFLLfLKlS9Lq9NbcFqTv7F29VqeevlRzvn7GZx69klced6N299BMVBYDzAY8KiZtQCygFr8PnVX18wmx8vlgZF57sSsDNAMGBLNCAJQAsDdl5jZg8BwouD7LWHTN8xsQ7xcKtc+axEF1zu5DtcFaArsk1c97t4X6AtQouQ+nle/oiY5OZm2bU/nuGZnhi6lSKtetTLVqlSmweGHAHBqyxN4eeBbVKtamZNPPB4z44jDDsbMWLlqNRX3rsC/brs+Z/tLr7+DOvvUClV+kbdk0VIAViz/jY8/+JyGjepz/sXn0PnexwEY+u7HPPF0l4AVFh/HtmjCgl8WsnJFNC3/+QcjuPHuaylZsgTDxrwNQMlSJRk2ZghnHvv3kKXusIJ+rJsONNpG+6VAFaBRPMpZApSM181x94Zx+135qGdVdv/436EJ648AVgA1cx8/4Rgbcq3rDDwM5A6TTUQjo/u2U1Oxc1Lr5vw4cw4LFiwOXUqRVrlSRapXrcLP8+YDMGbCZOrW2ZfWzY9j3MQpAMz9ZT5bMjLYu0J5NmzcyPoNGwH4etxEUpKTqbv/fsHqL8pKlS7FXmVK5yw3b9WMH2fMZsniZRx7fDSrfnyLY5g7Z1sTFpLbogVLaHB0fUqWKgHAMc0b81rvQbQ84ixOa3IepzU5j40bNhbbIIKCj4y+IBoBXRePGjCzBsB+wFJ332JmreLXBRZP8/1sZn939yHxAxMN3H2KmTUFzgCOAkaa2Sfu/vN2dlkXSHP3T8ysWa51jwFTgXFmNtjdp+9IzSG99tpztGh+LJUrV2TO7HE83O0pBgwYzN8vOEcPLuRTpw7t+VfX7mzJ2MI+NWvwcKcOlC5Vkvsf7cm5l91AamoKj95/J2bGbytXc32H+7CkJKpVqcRjD3YMXX6RVaVKJfq+Ht0PSklJ5t23hzHy86+4J70LXR67h+SUZDZt2sQ9HboGrrR4mDpxOp8O/YK3Pn2VzMxMfpg6kyGvvxu6rEJl7gWbfTKzmkSPdjciulc0l2jK6xmgDDCeaFrsjHiToe5eP962HdDG3a9I2F+6u5dJeL0/8CJQA0gF3gSeAMYBV7r7RDM7B+hAdC9pONDR3ccn7s/MWsbrjnP3MWbWBUh39yfNbET2NmZ2PNAdaO7uWXmdd3Gapivq0ufnOVMrO6BuvbahS9itlEvdK3QJu5VpS8bY9nvtQBjtqRRGhUdhVLgURoVLYVS48htGehRIRESCUxiJiEhwCiMREQlOYSQiIsEpjEREJDiFkYiIBKcwEhGR4BRGIiISnMJIRESCUxiJiEhwCiMREQlOYSQiIsEpjEREJDiFkYiIBKcwEhGR4BRGIiISnMJIRESCUxiJiEhwCiMREQlOYSQiIsEpjEREJLiU0AUUF5lZWaFL2G30OerB0CXsViY2qRi6hN1Kk/GrQpewR9LISEREglMYiYhIcAojEREJTmEkIiLBKYxERCQ4hZGIiASnMBIRkeAURiIiEpzCSEREglMYiYhIcAojEREJTmEkIiLBKYxERCQ4hZGIiASnMBIRkeAURiIiEpzCSEREglMYiYhIcAojEREJTmEkIiLBKYxERCQ4hZGIiASXEroAKRz16tXlP2+8mPP6gP33pUvXJ3nm2ZcDVlW0lalRkZN73UDpyuVxd6b/Zzjf9fuYph3+xmGXtGTDirUAjHniLeYNn8I+zetz3D0XkpyWQubmDL56ZBALvv4+8FkUMUlJVHi2L1krlrHmwXspec55lDqvHck1a7Pi7+fga1Zv1T2l3iGU7/U8ax99iM2jRwYqungoW64sTzzdmXqHHoi7c/ctnTm9zUmcdPqJbNm8hXlz53PXzQ+yds3a0KXukEILIzM7BngcqAikAaOAO9w9vbCOIXmbOXMOjZucCkBSUhK/zJ3Au+99GLiqoi0rM4uvHv4Py6bNJXWvklw47GF+HTUVgCkvf8SkPsO26r/ht7V8cNVTrFuyiooH1+acgXczoMmtIUovskqe246MX+eRVLo0AFumT2Pz2G8o373XHzsnJVH66uvZMmH8Lq6yeOr82N2M/PwrbryyI6mpKZQsVYrRI8bQ/eFnyMzM5F+db+fGDlfzRNdtXOtioDCn6UoC/3D3I939UGASoI/lAZzU+gR++mkev/yyIHQpRdr6patYNm0uAFvWbeS32QspU71inv2XT5/HuiWrAPjtx/mklEwjKU2TC9mSKlchremxbPpwaE5b5pxZZC1ZvM3+Jdv+jc2jR5K1auWuKrHYKlu2DE2Pa8Tgge8AsGVLBmvXrGXUiG/IzMwEYNL476heo2rIMndKoYWRu4909/kJr18E6plZXTNraWarzWyymS02s44AZtbIzEaa2QQz+9jMasTtI8yssZklm9n7ZnalmTWPt//ezDbEy5Pj/nPNrLuZTTWzcWZ2YNw+wMzaxcvXmJmbWWUzq2Nm0+L2Q81sipntU1jXIrQLLmjLm4PfDV1GsVK2dmWqHL4fiyfNAeCIf57CRZ88Susnr6VE+dJ/6F/3zCYsmzqXrM0Zu7rUImuvG25m3cu9wX27fZMqVaZEs+ZsHPreLqis+Ku9Xy1+W7GSHs89xNDhg3m8V2dKlS61VZ8LLjmXkZ9/FajCnVeoDzCY2V3ZIREHxQHAYUAyMNLdGwK9476pwLNAO3dvBPQDHsm1yz7AGHfv7+6j4u3PBOa4e8P4dbbV7n4E8Byw1TjVzEoCNwBLc7XXAgYBl7j7r9s4n+vMbLyZjc/KWrdjF2UXS01N5ew2p/L2f4duv7MAkFq6BGf0uY1RXQayJX0DU1//jNdPuIM3T7uP9UtXcfwDl27Vv2K9WjTrdBHD7+0XqOKiJ/WY48hatYrM2TPz1X+vG25h3St98hVcAikpyRze4BDe6D+ENq0uZP36DbS/7aqc9TfdcQ0ZmZm8O+SDgFXunEKdY3D3HkCP7Ndm9mW8WArYmKv7wUB94FMzgyiwFiWs7wI0BfI7YhmU8N+eudbdBLwK3JnQVgb4CPjC3adva4fu3hfoC5CSVqtY/NScfnorJk2aytKly0OXUiwkpSRzRt/bmPnu1/z0UXTvYsPyNTnrp/9nOG0G/P5ts1f1ipz50u18entv1sxb+of97alSD6tP2rHNSGtyDJaWhpXeizJ330d699yfLyMp9Q6m7L0PApBUvjxpTY8lPTOTzd+M3pVlFxuLFi5h8cIlTJ4Q3dP88P1PuSEOo/MvPofWp7bg0vOuC1niTvvLJrzNrBzQEPgeOAlYmLsLMN3dj8tjF5uIRkb3AQ/m45Cex3I54CLgeLYOo32Ay4B7zexQd5+Rj2MUeRddeK6m6AqgdY9r+G3WQia/9PvDHqWrVmD90uje0AGnN2bFj9Hsc1q50pz96p18/dhgFo+fFaTeomp9/5dY3/8lAFIbNKRUuwvzDCKAlf+8KGe5zJ33sHnsNwqiP7F86QoWLVjCAQfux0+z59GsxTHM/vEnWrRuxvW3XMFFZ1/Nxg25P+8XL4X5NN0VwBR3n2RmycBTRCOPucDfgFdybfIjUMXMjnP3b+Jpu3oJo5THgKnAODMbnNfoJcGFRE/zXQh8k9DeAejh7pvjEVi2Ge4+yMwWAn3M7ET34j1nULp0KU4+qQXtb/xX6FKKhRpN6nFIu+Ysn/ELF34UvXGOeeItDmp7HFUO3w93Z+385Qy/J5qOa3DFKZSvU40mt59Hk9vPA+D9S59gw4o1eR5jT1ey7fmU+vtFJFWsSIXe/dgybgzpvXpsf0P5g873PE7PPo+RlprKL/Oix7jf++w/pJVI4/X/9gZg0vip3N+xW+BKd4wV1vuvmTUBugPliR7v/gy4g+ge0QrgNnfPMrMuQLq7P2lmDYFn4m1SgF7u/pKZjQA6uvt4Mzs+3m/zePs6wFB3r59w7LnAYOAMohHVxe4+28wGAI2BBvG2c+PXZRL3YWZ9gcnu/kJe51dcpumKg17VWoUuYbdy0WF/uN0pO6HJ+FWhS9it/Lxiim2/VyGGUUjZIePuf9mNEoVR4VEYFS6FUeFSGBWu/IaR/hyQiIgEt1v8xp671wldg4iI7DiNjEREJDiFkYiIBKcwEhGR4BRGIiISnMJIRESCUxiJiEhwCiMREQlOYSQiIsEpjEREJDiFkYiIBKcwEhGR4BRGIiISnMJIRESCUxiJiEhwCiMREQlOYSQiIsEpjEREJDiFkYiIBKcwEhGR4FJCF1BcJJmFLmG3sTrJQ5ewW/l+StXQJexW3kirHLqEPZJGRiIiEpzCSEREglMYiYhIcAojEREJTmEkIiLBKYxERCQ4hZGIiASnMBIRkeAURiIiEpzCSEREglMYiYhIcAojEREJTmEkIiLBKYxERCQ4hZGIiASnMBIRkeAURiIiEpzCSEREglMYiYhIcAojEREJTmEkIiLBKYxERCS4lNAFyI7r2+dJzjzzZJYtW85RR5+81brbb7+O7k88SI2aR7BixcpAFRZtZWtU5Jye7dmrcnlwZ9J/vuDb/h/TutPFHHTS0WRuyWDVvCX8766+bFqznqSUZM564hqq19+fpJQkpv53NF+/8H7o0ygyrEQqR777EElpKVhKMsuHjmFej7c45PlbKXNkXTwjg7WTZjPrrr54RiZV/3YCtW8+FzMjI30Ds//1Euu+nxf6NIoEK5FK/XceJiktFUtJZsXQb/j1ycEc9PxtlGlQF8/IZO2kWfx0dx88I5PksqU56LnbKFGrMpaSzMIX32Pp4OGhT6NAzN3/vIOZA2+4+2Xx6xRgETDW3dv89SUWDWklav/5hQrghBOOIT19Hf379doqjGrXrkHv3j04uN6BHHvcGUUujDpXPzF0CQCUqVqBMlUrsHjaXNL2KslVQ7vx9nU9KVu9InO/no5nZtHqnosAGP74mxzethkHnXw0797yHCkl07j+s+4MvKgbq+cvD3oezTdtDnr8REmlS5K1fiOWksyR7z/MnPv7k7J3GVZ+PgmAQ168jdVjZrDo1U8o17ge62ctIGP1OvZu3ZCstahVAAAPw0lEQVT9Ol7A5DM7BT4DSLGs0CUAW1/L+u914+cH+pFSoSyrvpgIwEEvdGDNmO9Z8trH1Lr1b6SULc28RwaSUqkcR416hvFHXoNvyQh8FtBs0X8tP/3yM023DqhvZqXi16cAC3a0MCk8o0ePZeXKVX9of7JHFzrd+wjb+6Cxp0tfuorF0+YCsHndRlbMXkjZanvz86ipeGb0hrRw0mzK1agIgLuTVroElpxEask0MrdksGnthlDlF0lZ6zcCYKnJWEoyuOcEEcDaSbMpUaMSAGvGzyRj9bqofcKsnHaJbHUtU1PAyQkigPTJsyhRM75m7iSXid6ik0uXJGNVOp6Ructr3hn5vWc0DDgrXr4YGARgZklmNsvMqiS8nm1mVczsYTN7PG7vYmYd4+XnzOzO3AdI7BO/HmpmLePlF81svJlNN7Ou2yrQzOaaWeVtHO9aM/vWzKaY2X/NrHTcXs3M3onbp5hZs3xeiyLt7LNPZcHCxXw3dUboUoqV8rUrU+3w/Vgwec5W7UdecCJzRkwB4Idh49i8fhO3ffs8N3/zNGP7fsDG+M1UYklJHP1ZD46b9gqrvvyOtZNm56yylGSqtmvBb8Mn/WGz6pe05rcv/ti+R0tK4shPn6TJ1H6sHjmF9EmzclZZSjJV2p3IyvhaLur3IaUOqk3jyS/TcPi/mftAPyhmH0bzG0ZvAheZWUmgATAWwN2zgIHApXG/k4Ep7r4MeBCoY2ZXZe/EzG4DSrj7UwWs8z53bxwf+0Qza7CNPlnAtoaD/+fuTdz9SGAGcHXc/gwwMm4/GphewJqKnFKlSvKvu2+ha9cnQ5dSrKSWLsH5vW/n04deZ3P67yOd429uS1ZGJtPe+QqAmg3r4llZPNP0Zp4/oQPHXHsmFfapEqrsoikri4kn38WYo66n7FEHUvqQfXJWHfj4NaweM4M1Y3/YapPyxx9O9Ytb83O3gbu62qItK4spp3Rk/NHXUeaogyh98O/X8oDHr2XNmO9ZOzb60Ll3y4asm/4z4xtew5STO7L/o9fkjJSKi3yFkbt/B9QhGhUNy7W6H3B5vHwV0D/exoEuwItAc+A0oAfw0J8cqoOZTTazyfE22S4ws4nAJOBw4LBtbDsfOGob7fXNbJSZTSUKzcPj9tZxbbh7pruvzr2hmV0Xj8jGZ2UW/U/AdQ+oQ506+zD+20+Y+eM31K5dg7FjPqJaNb1h5iUpJZnze9/OtHe/4sePxue0N2jXggNPOop3b3shp+3wts2YM+I7sjIyWb9iDfMnzKRGgwNClF3kZa5Zz6qvplOxVUMA9r2zHamVyvFT51e36rfXoftS76kbmH5FdzJWpocotcjLXLOe1V9No0Kr6O2t9h1/J6VSeeZ2HpDTp+pFrVkxbCwAG+cuZtMvSyl1YK0Q5e6wgjza/T7wJPEUXTZ3/xVYYmatgabAhwmrnyAKqLrAPsANwON/coye7t7Q3RsCowDMbH+gI3CSuzcAPgBKbmPbTkD3OMhuSGgfANzs7kcAXfPYdpvcva+7N3b3xknJe+V3s2CmTf+B2vs0pN7Bx1Hv4OOYP38Rxxx7OkuWLAtdWpF1VvdrWTF7AeNe/v3b9oATG3DsDW0YcvVTZGz8/eGANQuWU6dZ9DkotVQJah51ECvmLNzlNRdVqZXKkVyuNABJJdPYu0UD1s9eQPVLWrN3y4b80P7praaOStSqzGH97uLHm59lw0+LQpVdJKXkupYVTmzAhtkLqHrJSVRo2ZBZ7XtudS03LVhOhROOACC1cnlK1q3Jxl+WBKl9RxXk0e5+wCp3n5p9LyfBy0TTda+7eyaAmZ0DbHT3N8zsICDd3fuZ2flm1srd8/vcYTmihyhWm1k14AxgRO5O7j4aaBgfu0vCqrLAIjNLJRoZZT988TnQHuhlZslAmW2Njoqy1197jhYtjqNy5Yr8NOdbHnr4KQYMeDN0WcVG7cb1aHB+c5bM+IVrhj0KwPAegzm1y+WkpKVyycB7AVgwaTYf3teP8a99ytlPXs91nz4BZnw3ZCRLf/g15CkUKWlVK3DwMzdDchKWZCx7/xt++3Qizee/ycb5y2g49BEAlg8byy//fpt972hHyt5lOPDxawHwzEwmnXZPyFMoMtKq7s2BT9+MJSdjScby979m5WcTOO7Xt9g0fxlH/C/6fl0xbCzzew7h155DOOjpmznyi39jZsx7ZCAZv60NfBYFk59Hu9PdvUyutpZAx+xHu+M3+hVAU3f/IX7ybixwursvjMMh3d2fNLMDgLeBY919c8I+c/rEr4cCT7r7CDMbADQDfgVWA++7+4A/qTnxeO2Bu4FlcU1l3f2KONj6AgcAmUB7d/8mr30WxUe7i6ui8mj37qIoPdq9Oygqj3bvLvL7aPd2wyhfOzFrTDTF1ny7nYsphVHhURgVLoVR4VIYFa78htFO/wUGM7uHaLrr0u31FRER2Zad/tt07v64u+8X37MREREpMP2hVBERCU5hJCIiwSmMREQkOIWRiIgEpzASEZHgFEYiIhKcwkhERIJTGImISHAKIxERCU5hJCIiwSmMREQkOIWRiIgEpzASEZHgFEYiIhKcwkhERIJTGImISHAKIxERCU5hJCIiwSmMREQkOIWRiIgEZ+4eugYpRGZ2nbv3DV3H7kLXs/DoWhau3e16amS0+7kudAG7GV3PwqNrWbh2q+upMBIRkeAURiIiEpzCaPez28whFxG6noVH17Jw7VbXUw8wiIhIcBoZiYhIcAojEREJTmFUxJjZNDP73swmm9kCM+sSuqaixsyqm9mbZjbHzCaY2TAzqxe6ruLOzI4xs+FmNsXMZphZXzMrE7qu4sDM3MwGJrxOMbNlZjY0ZF3FicKoaDrD3RsCPUMXUtSYmQHvACPcva67NwLuBaqFrWy3UBL4h7sf6e6HApOAlwPXVFysA+qbWan49SnAgoD1FDsKo6InFdiU2GBmLc3sSzP7wMx+NLPeZpZkZleZWa+EfteaWU8zq2NmG+LR1WQzey1eP9fMKsfLA81sWrycbGY9zOxbM/vOzK7fhedbUK2ALe7eO7vB3acAk8zsczObaGZTzawtQHwtpmX3NbN2ZjbAzOomXJ/MhOWa8bqP4lHXKDM7JN72PTO7PF6+3szeiJdHmFnjhGOkx/9tmf3J2MwqmtkqM+uYuE187d83syv/6gu3Pe4+0t3nJ7x+EagXX4+WZrY6vkaLE86jkZmNjK/Vx2ZWI27/w/mZWfN4++8Tvz/j/nPNrHv8tRtnZgfG7QPMrF28fE08Aqmc+HU1s0Pj0dw+u/iS5TYMOCtevhgYBBD/rM4ysyoJr2ebWRUze9jMHo/buyRc1+fM7M7cB0jsE78eamYt4+UXzWy8mU03s67bKjDXe0Di8a6Nf/6nmNl/zax03F7NzN6J26eYWbPCuFDbojAqesoCa7fR3hS4BTgMqAv8DXgLONvMUuM+VwL94uU57t4w/nd54o7M7AigfkLT1cBqd28CNAGuNbP9C+uECll9YMI22jcC57n70USB9ZSZWV47cfec6wNsSLhWC4kemb0lHnV1BF6IN7sOeNDMmgN3En098ute4JdttPcBxrh7/wLs6y9jZnclBPNk4ACi77lkYGR8vXrHfVOBZ4F28bXqBzySa5c55+fuo+LtzyTh+zOh72p3PwJ4DuiVuBMzKwncACzN1V6L6E3/Enf/tTCuwU54E7gorrUBMBbA3bOAgcClcb+TgSnuvgx4EKhjZldl78TMbgNKuPtTBTz+fe7eOD72iWbWYBt9soBt/Vz8n7s3cfcjgRlE7wkAzxB93Y8EjgamF7CmfEv5q3YsBWdmyUBZd1+3jdXj3P2nuN8g4AR3f9vMvgDamNkMINXdp5pZne0cqhvQmd/fOE4FGmR/AgXKAwcBP+/UCe1aBjxqZi2IfuBq8fvUXd3sT+BE5zYyz51E90iaAUMSsqwEgLsvMbMHgeFEwfdbwqZvmNmGeLlUQnv2G+axRNOLiboQfcgI/Yk+h7v3AHpkvzazL+PFUkSBn+hgog8Hn8bXKhlYlLC+CwU7v0EJ/809RX0T8CrRh4BsZYCPgC/c/S97k8wvd/8u/tm7mGiUlKgf8B5RyF4F9I+3cYvuC08BRhN9755I9IEzLx3M7LJ4eX/gyXj5AjO7juh9vQbRh4jvcm07HzgK+CRXe30z6wZUILquH8ftrYHL41ozgdV/UtdO0cioaDkAmJnHuty/EJb9+mXgCqJRUX4+XTcD0om++bMZ0Ugge3Swv7vn/mYtKqYDjbbRfilQBWgUf9peQnQPBLb+FH7XdvafBKxKuBYN4/sn2Y4AVgA1cx8/caSVa11n4GH++DXcRDRyuG87NQVhZuWAhsD3ROe7MHcXYHrCdTrC3U9NWF/Q8/M8lssBF8X7SrQP8CjQyswOpWh4nygcBiU2xqO2JWbWmiigP0xY/QRRQNUlOqcbgMf/5Bg9E77XRgHEMxkdgZPcvQHwAb9//yfqBHSPP5zdkNA+ALg5Hpl2zWPbv5TCqGi5APgmj3VNzWx/M0sCLiT6FIW7jyX6Br6EXD8AeehCNDWQ6GOgffZ0n5nVM7O9Cl7+LvEFUCL+BAhAPB2xH7DU3beYWav4dYG5+xrgZzP7e7xvM7Mj4+WmwBlEnyw75nMqsy5QJ49wf4xolNrWzA7fkXoLk5ldYWZHxcvJwFNEI4+5RNPCX+Xa5EegipkdF2+Tmus8Cnp+Fyb8N/HnoAPwrLtvztV/hrsPIpou7fNn07K7UD+gq7tP3ca6l4mm64bEowzM7Bxgo7u/QRQIL7t7P6BC/H2cX+WIHqJYbWbViL5P/8DdRycEWe+EVWWBRfF7wKUJ7Z8D7eNak82sfAFqKhCFURFhZu2JfnDbJMzXdyD69FIO+JZoLn0G0fRZ4pTPW8BX7r4yH4ca6+5zcrW9TPTpd2J8U7gPRXQK16M/GXIecLJFj3ZPJ3rTGwY0NrOpRNMKP+zEYS4FrjazKUQjsbZmVgJ4Cbgqvq90J9AvH2+Ah/DH8E88n03AjUDf+INGSNOBf5vZRGAO0cjnGuB1YBbw38TOcTi0A56Ir9VkopF3Yp+CnN/eZvYdcBvR9342I3oT3yZ3H0n09W6/vRP8q7n7fHd/Jo/V7xNNgfUHsOjJu25sfa7ZbiG675mWz+NOIXr68QfgP/zxg8P2PEB0j+srtv7ZuY1o5DmV6F7tYQXcb77pzwEVEfG88Vx3H7CtdqKbxG3y2HYo0dD987+2SpG/hpnNBRq7+/LQtfxVLHrisqe7Nw9dS1EU+pOY7AQzq2BmM4meBlMQiRRRZnYP0cjy3tC1FFUaGRURZpZCNAuVmZ92EZHdicJIRESC0zSdiIgEpzASEZHgFEYiIhKcwkhERIL7f7LwxRkY+b4yAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "print_result(stack, test_text, X_test, y_test, grid, True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}