ML Pipeline Preparation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ML Pipeline Preparation\n",
    "Follow the instructions below to help you create your ML pipeline.\n",
    "### 1. Import libraries and load data from database.\n",
    "- Import Python libraries\n",
    "- Load dataset from database with [`read_sql_table`](https://pandas.pydata.org/pandas-docs/stable/generated/pandas.read_sql_table.html)\n",
    "- Define feature and target variables X and Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
      "[nltk_data]     /root/nltk_data...\n",
      "[nltk_data]   Unzipping taggers/averaged_perceptron_tagger.zip.\n",
      "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
      "[nltk_data]   Unzipping tokenizers/punkt.zip.\n",
      "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
      "[nltk_data]   Unzipping corpora/wordnet.zip.\n",
      "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
      "[nltk_data]   Unzipping corpora/stopwords.zip.\n"
     ]
    }
   ],
   "source": [
    "# import libraries\n",
    "\n",
    "import pandas as pd\n",
    "from sqlalchemy import create_engine\n",
    "import re\n",
    "import numpy as np\n",
    "import pickle\n",
    "from collections import defaultdict\n",
    "from collections import Counter\n",
    "import unicodedata\n",
    "import string\n",
    "\n",
    "import nltk\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.tokenize import sent_tokenize\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "nltk.download('averaged_perceptron_tagger')\n",
    "nltk.download('punkt')\n",
    "nltk.download('wordnet')\n",
    "nltk.download('stopwords')\n",
    "\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.multioutput import MultiOutputClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.pipeline import FeatureUnion\n",
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.preprocessing import LabelBinarizer\n",
    "from sklearn.decomposition import PCA, TruncatedSVD\n",
    "\n",
    "from gensim.models.doc2vec import TaggedDocument, Doc2Vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load data from database\n",
    "engine = create_engine('sqlite:///InsertDatabaseName.db')\n",
    "df = pd.read_sql_table('disaster_response', engine)  \n",
    "X = df.iloc[:,0:4]\n",
    "Y = df.iloc[:,5:40]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Write a tokenization function to process your text data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenize(text):\n",
    "    \"\"\"\n",
    "    This function tokenizes, lemmatizers, and lowercases text, removes stopwords and punctuation\n",
    "\n",
    "    Arguments:\n",
    "        x: string or Pandas series \n",
    "\n",
    "    Returns:\n",
    "        cleaned tokens\n",
    "    \"\"\"\n",
    "    \n",
    "    tokens = word_tokenize(text)\n",
    "    tokens = [t for t in tokens if t.lower() not in stopwords.words(\"english\") \n",
    "              and t.lower() not in string.punctuation]\n",
    "    lemmatizer = WordNetLemmatizer()\n",
    "\n",
    "    clean_tokens = []\n",
    "    for tok in tokens:\n",
    "        clean_tok = lemmatizer.lemmatize(tok).lower().strip()\n",
    "        clean_tokens.append(clean_tok)\n",
    "\n",
    "    return clean_tokens"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Build a machine learning pipeline\n",
    "This machine pipeline should take in the `message` column as input and output classification results on the other 36 categories in the dataset. You may find the [MultiOutputClassifier](http://scikit-learn.org/stable/modules/generated/sklearn.multioutput.MultiOutputClassifier.html) helpful for predicting multiple target variables."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipeline = Pipeline([\n",
    "        ('vect', CountVectorizer(tokenizer=tokenize)),\n",
    "        ('tfidf', TfidfTransformer()),\n",
    "        ('clf', MultiOutputClassifier(RandomForestClassifier()))\n",
    "    ])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4. Train pipeline\n",
    "- Split data into train and test sets\n",
    "- Train pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y = Y.drop('child_alone', axis = 1) # remove this variable as it has only 0s\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(memory=None,\n",
       "     steps=[('vect', CountVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
       "        dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
       "        lowercase=True, max_df=1.0, max_features=None, min_df=1,\n",
       "        ngram_range=(1, 1), preprocessor=None, stop_words=None,\n",
       "        strip...oob_score=False, random_state=None, verbose=0,\n",
       "            warm_start=False),\n",
       "           n_jobs=1))])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipeline.fit(X_train['message'], y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5. Test your model\n",
    "Report the f1 score, precision and recall for each output category of the dataset. You can do this by iterating through the columns and calling sklearn's `classification_report` on each."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = pipeline.predict(X_test['message'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "category_names = list(Y.columns)\n",
    "y_pred = pd.DataFrame(y_pred)\n",
    "y_pred.columns = category_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.89      0.98      0.93      5423\n",
      "          1       0.80      0.44      0.57      1157\n",
      "\n",
      "avg / total       0.88      0.88      0.87      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       1.00      1.00      1.00      6553\n",
      "          1       1.00      0.04      0.07        27\n",
      "\n",
      "avg / total       1.00      1.00      0.99      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.74      0.86      0.79      3841\n",
      "          1       0.74      0.57      0.64      2739\n",
      "\n",
      "avg / total       0.74      0.74      0.73      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.92      0.99      0.96      6037\n",
      "          1       0.60      0.09      0.16       543\n",
      "\n",
      "avg / total       0.90      0.92      0.89      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.95      1.00      0.97      6229\n",
      "          1       0.77      0.07      0.13       351\n",
      "\n",
      "avg / total       0.94      0.95      0.93      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.97      1.00      0.99      6404\n",
      "          1       0.48      0.06      0.11       176\n",
      "\n",
      "avg / total       0.96      0.97      0.96      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.98      1.00      0.99      6451\n",
      "          1       0.50      0.01      0.02       129\n",
      "\n",
      "avg / total       0.97      0.98      0.97      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.97      1.00      0.98      6355\n",
      "          1       0.57      0.08      0.13       225\n",
      "\n",
      "avg / total       0.95      0.97      0.95      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       1.00      1.00      1.00      6580\n",
      "\n",
      "avg / total       1.00      1.00      1.00      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.95      1.00      0.97      6139\n",
      "          1       0.90      0.24      0.37       441\n",
      "\n",
      "avg / total       0.94      0.95      0.93      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.93      0.99      0.96      5854\n",
      "          1       0.82      0.44      0.57       726\n",
      "\n",
      "avg / total       0.92      0.93      0.92      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.93      0.99      0.96      5988\n",
      "          1       0.79      0.26      0.39       592\n",
      "\n",
      "avg / total       0.92      0.93      0.91      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.99      1.00      0.99      6480\n",
      "          1       0.68      0.15      0.25       100\n",
      "\n",
      "avg / total       0.98      0.99      0.98      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.98      1.00      0.99      6421\n",
      "          1       1.00      0.02      0.04       159\n",
      "\n",
      "avg / total       0.98      0.98      0.97      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.99      1.00      0.99      6498\n",
      "          1       1.00      0.01      0.02        82\n",
      "\n",
      "avg / total       0.99      0.99      0.98      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.97      1.00      0.98      6366\n",
      "          1       0.52      0.05      0.09       214\n",
      "\n",
      "avg / total       0.95      0.97      0.95      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.97      1.00      0.98      6306\n",
      "          1       0.79      0.17      0.28       274\n",
      "\n",
      "avg / total       0.96      0.96      0.95      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.87      1.00      0.93      5707\n",
      "          1       0.70      0.05      0.09       873\n",
      "\n",
      "avg / total       0.85      0.87      0.82      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.94      1.00      0.97      6171\n",
      "          1       0.29      0.00      0.01       409\n",
      "\n",
      "avg / total       0.90      0.94      0.91      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.96      1.00      0.98      6295\n",
      "          1       0.57      0.08      0.14       285\n",
      "\n",
      "avg / total       0.94      0.96      0.94      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.96      1.00      0.98      6249\n",
      "          1       0.80      0.16      0.26       331\n",
      "\n",
      "avg / total       0.95      0.96      0.94      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.98      1.00      0.99      6452\n",
      "          1       0.80      0.03      0.06       128\n",
      "\n",
      "avg / total       0.98      0.98      0.97      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.99      1.00      1.00      6545\n",
      "          1       0.00      0.00      0.00        35\n",
      "\n",
      "avg / total       0.99      0.99      0.99      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.99      1.00      0.99      6511\n",
      "          1       1.00      0.03      0.06        69\n",
      "\n",
      "avg / total       0.99      0.99      0.99      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       1.00      1.00      1.00      6556\n",
      "          1       0.00      0.00      0.00        24\n",
      "\n",
      "avg / total       0.99      1.00      0.99      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.99      1.00      0.99      6504\n",
      "          1       0.00      0.00      0.00        76\n",
      "\n",
      "avg / total       0.98      0.99      0.98      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.96      1.00      0.98      6300\n",
      "          1       0.00      0.00      0.00       280\n",
      "\n",
      "avg / total       0.92      0.96      0.94      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.86      0.95      0.90      4758\n",
      "          1       0.82      0.60      0.69      1822\n",
      "\n",
      "avg / total       0.85      0.85      0.84      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.95      0.99      0.97      6055\n",
      "          1       0.86      0.37      0.52       525\n",
      "\n",
      "avg / total       0.94      0.94      0.93      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.94      0.98      0.96      5949\n",
      "          1       0.74      0.41      0.53       631\n",
      "\n",
      "avg / total       0.92      0.93      0.92      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.99      1.00      0.99      6512\n",
      "          1       1.00      0.03      0.06        68\n",
      "\n",
      "avg / total       0.99      0.99      0.99      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.97      0.99      0.98      5972\n",
      "          1       0.88      0.67      0.76       608\n",
      "\n",
      "avg / total       0.96      0.96      0.96      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.98      1.00      0.99      6446\n",
      "          1       0.78      0.19      0.30       134\n",
      "\n",
      "avg / total       0.98      0.98      0.98      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.95      1.00      0.97      6233\n",
      "          1       0.50      0.04      0.08       347\n",
      "\n",
      "avg / total       0.93      0.95      0.93      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.85      0.98      0.91      5262\n",
      "          1       0.78      0.31      0.45      1318\n",
      "\n",
      "avg / total       0.84      0.84      0.82      6580\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/sklearn/metrics/classification.py:1135: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "for i in category_names:\n",
    "    print(classification_report(y_test[i], y_pred[i]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 6. Improve your model\n",
    "Use grid search to find better parameters. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "parameters = {\n",
    "    'clf__estimator__n_estimators': [50, 100, 200],\n",
    "    'clf__estimator__min_samples_split': [2, 3, 4]\n",
    "}\n",
    "\n",
    "model = GridSearchCV(pipeline, param_grid=parameters, verbose = 5, n_jobs = -1, cv=2, scoring = 'roc_auc')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 7. Test your model\n",
    "Show the accuracy, precision, and recall of the tuned model.  \n",
    "\n",
    "Since this project focuses on code quality, process, and  pipelines, there is no minimum performance metric needed to pass. However, make sure to fine tune your models for accuracy, precision and recall to make your project stand out - especially for your portfolio!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 2 folds for each of 9 candidates, totalling 18 fits\n",
      "[CV] clf__estimator__min_samples_split=2, clf__estimator__n_estimators=50 \n",
      "[CV]  clf__estimator__min_samples_split=2, clf__estimator__n_estimators=50, score=0.7934432096983246, total= 3.6min\n",
      "[CV] clf__estimator__min_samples_split=2, clf__estimator__n_estimators=50 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   1 out of   1 | elapsed:  4.6min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  clf__estimator__min_samples_split=2, clf__estimator__n_estimators=50, score=0.7955468482257044, total= 3.6min\n",
      "[CV] clf__estimator__min_samples_split=2, clf__estimator__n_estimators=100 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   2 out of   2 | elapsed:  9.1min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  clf__estimator__min_samples_split=2, clf__estimator__n_estimators=100, score=0.8034745827085664, total= 5.5min\n",
      "[CV] clf__estimator__min_samples_split=2, clf__estimator__n_estimators=100 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   3 out of   3 | elapsed: 15.8min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  clf__estimator__min_samples_split=2, clf__estimator__n_estimators=100, score=0.8045407579060148, total= 5.5min\n",
      "[CV] clf__estimator__min_samples_split=2, clf__estimator__n_estimators=200 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   4 out of   4 | elapsed: 22.4min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  clf__estimator__min_samples_split=2, clf__estimator__n_estimators=200, score=0.8077783518234741, total= 9.4min\n",
      "[CV] clf__estimator__min_samples_split=2, clf__estimator__n_estimators=200 \n",
      "[CV]  clf__estimator__min_samples_split=2, clf__estimator__n_estimators=200, score=0.8125869734825578, total= 9.3min\n",
      "[CV] clf__estimator__min_samples_split=3, clf__estimator__n_estimators=50 \n",
      "[CV]  clf__estimator__min_samples_split=3, clf__estimator__n_estimators=50, score=0.795871942925752, total= 3.3min\n",
      "[CV] clf__estimator__min_samples_split=3, clf__estimator__n_estimators=50 \n",
      "[CV]  clf__estimator__min_samples_split=3, clf__estimator__n_estimators=50, score=0.7986815631676302, total= 3.3min\n",
      "[CV] clf__estimator__min_samples_split=3, clf__estimator__n_estimators=100 \n",
      "[CV]  clf__estimator__min_samples_split=3, clf__estimator__n_estimators=100, score=0.8081066333276922, total= 5.1min\n",
      "[CV] clf__estimator__min_samples_split=3, clf__estimator__n_estimators=100 \n",
      "[CV]  clf__estimator__min_samples_split=3, clf__estimator__n_estimators=100, score=0.8082096382584517, total= 5.1min\n",
      "[CV] clf__estimator__min_samples_split=3, clf__estimator__n_estimators=200 \n",
      "[CV]  clf__estimator__min_samples_split=3, clf__estimator__n_estimators=200, score=0.8127099937389812, total= 8.5min\n",
      "[CV] clf__estimator__min_samples_split=3, clf__estimator__n_estimators=200 \n",
      "[CV]  clf__estimator__min_samples_split=3, clf__estimator__n_estimators=200, score=0.8164218079383837, total= 8.5min\n",
      "[CV] clf__estimator__min_samples_split=4, clf__estimator__n_estimators=50 \n",
      "[CV]  clf__estimator__min_samples_split=4, clf__estimator__n_estimators=50, score=0.7993562816836846, total= 3.2min\n",
      "[CV] clf__estimator__min_samples_split=4, clf__estimator__n_estimators=50 \n",
      "[CV]  clf__estimator__min_samples_split=4, clf__estimator__n_estimators=50, score=0.803317262746898, total= 3.2min\n",
      "[CV] clf__estimator__min_samples_split=4, clf__estimator__n_estimators=100 \n",
      "[CV]  clf__estimator__min_samples_split=4, clf__estimator__n_estimators=100, score=0.8077110288309709, total= 4.8min\n",
      "[CV] clf__estimator__min_samples_split=4, clf__estimator__n_estimators=100 \n",
      "[CV]  clf__estimator__min_samples_split=4, clf__estimator__n_estimators=100, score=0.8098239794466616, total= 4.8min\n",
      "[CV] clf__estimator__min_samples_split=4, clf__estimator__n_estimators=200 \n",
      "[CV]  clf__estimator__min_samples_split=4, clf__estimator__n_estimators=200, score=0.8144858164286511, total= 8.0min\n",
      "[CV] clf__estimator__min_samples_split=4, clf__estimator__n_estimators=200 \n",
      "[CV]  clf__estimator__min_samples_split=4, clf__estimator__n_estimators=200, score=0.8142000277119759, total= 8.1min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  18 out of  18 | elapsed: 124.7min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: 0.814566 using {'clf__estimator__min_samples_split': 3, 'clf__estimator__n_estimators': 200}\n"
     ]
    }
   ],
   "source": [
    "model.fit(X_train['message'], y_train)\n",
    "print(\"Best: %f using %s\" % (model.best_score_, model.best_params_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = model.predict(X_test['message'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.91      0.98      0.94      5492\n",
      "          1       0.84      0.51      0.63      1088\n",
      "\n",
      "avg / total       0.90      0.90      0.89      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       1.00      1.00      1.00      6549\n",
      "          1       1.00      0.03      0.06        31\n",
      "\n",
      "avg / total       1.00      1.00      0.99      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.79      0.85      0.82      3861\n",
      "          1       0.76      0.68      0.72      2719\n",
      "\n",
      "avg / total       0.78      0.78      0.78      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.92      1.00      0.96      6042\n",
      "          1       0.80      0.07      0.13       538\n",
      "\n",
      "avg / total       0.91      0.92      0.89      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.95      1.00      0.98      6236\n",
      "          1       0.81      0.10      0.18       344\n",
      "\n",
      "avg / total       0.95      0.95      0.93      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.98      1.00      0.99      6413\n",
      "          1       0.71      0.09      0.16       167\n",
      "\n",
      "avg / total       0.97      0.98      0.97      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.98      1.00      0.99      6467\n",
      "          1       0.50      0.01      0.02       113\n",
      "\n",
      "avg / total       0.97      0.98      0.97      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.97      1.00      0.98      6378\n",
      "          1       0.53      0.04      0.08       202\n",
      "\n",
      "avg / total       0.96      0.97      0.96      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       1.00      1.00      1.00      6580\n",
      "\n",
      "avg / total       1.00      1.00      1.00      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.96      1.00      0.98      6158\n",
      "          1       0.87      0.36      0.51       422\n",
      "\n",
      "avg / total       0.95      0.96      0.95      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.95      0.98      0.97      5835\n",
      "          1       0.83      0.59      0.69       745\n",
      "\n",
      "avg / total       0.94      0.94      0.93      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.95      0.99      0.97      6010\n",
      "          1       0.78      0.39      0.52       570\n",
      "\n",
      "avg / total       0.93      0.94      0.93      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.98      1.00      0.99      6468\n",
      "          1       0.90      0.08      0.15       112\n",
      "\n",
      "avg / total       0.98      0.98      0.98      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.98      1.00      0.99      6419\n",
      "          1       0.83      0.03      0.06       161\n",
      "\n",
      "avg / total       0.97      0.98      0.97      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.99      1.00      1.00      6517\n",
      "          1       0.00      0.00      0.00        63\n",
      "\n",
      "avg / total       0.98      0.99      0.99      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.97      1.00      0.98      6385\n",
      "          1       0.50      0.02      0.03       195\n",
      "\n",
      "avg / total       0.96      0.97      0.96      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.96      1.00      0.98      6290\n",
      "          1       0.78      0.12      0.21       290\n",
      "\n",
      "avg / total       0.95      0.96      0.95      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.87      1.00      0.93      5724\n",
      "          1       0.67      0.03      0.05       856\n",
      "\n",
      "avg / total       0.85      0.87      0.82      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.93      1.00      0.96      6119\n",
      "          1       0.20      0.00      0.00       461\n",
      "\n",
      "avg / total       0.88      0.93      0.90      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.96      1.00      0.98      6276\n",
      "          1       0.85      0.12      0.20       304\n",
      "\n",
      "avg / total       0.95      0.96      0.94      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.95      1.00      0.97      6204\n",
      "          1       0.82      0.12      0.21       376\n",
      "\n",
      "avg / total       0.94      0.95      0.93      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.98      1.00      0.99      6454\n",
      "          1       0.83      0.04      0.08       126\n",
      "\n",
      "avg / total       0.98      0.98      0.97      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.99      1.00      1.00      6543\n",
      "          1       0.00      0.00      0.00        37\n",
      "\n",
      "avg / total       0.99      0.99      0.99      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.99      1.00      0.99      6497\n",
      "          1       0.00      0.00      0.00        83\n",
      "\n",
      "avg / total       0.97      0.99      0.98      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       1.00      1.00      1.00      6552\n",
      "          1       0.00      0.00      0.00        28\n",
      "\n",
      "avg / total       0.99      1.00      0.99      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.99      1.00      0.99      6503\n",
      "          1       0.00      0.00      0.00        77\n",
      "\n",
      "avg / total       0.98      0.99      0.98      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.95      1.00      0.98      6274\n",
      "          1       0.00      0.00      0.00       306\n",
      "\n",
      "avg / total       0.91      0.95      0.93      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.89      0.95      0.92      4742\n",
      "          1       0.85      0.69      0.76      1838\n",
      "\n",
      "avg / total       0.88      0.88      0.88      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.96      1.00      0.98      6052\n",
      "          1       0.91      0.49      0.64       528\n",
      "\n",
      "avg / total       0.95      0.96      0.95      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.95      0.99      0.97      5985\n",
      "          1       0.77      0.51      0.62       595\n",
      "\n",
      "avg / total       0.94      0.94      0.94      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.99      1.00      0.99      6503\n",
      "          1       0.50      0.01      0.03        77\n",
      "\n",
      "avg / total       0.98      0.99      0.98      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.98      0.99      0.98      5966\n",
      "          1       0.89      0.79      0.84       614\n",
      "\n",
      "avg / total       0.97      0.97      0.97      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.98      1.00      0.99      6447\n",
      "          1       0.83      0.08      0.14       133\n",
      "\n",
      "avg / total       0.98      0.98      0.97      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.95      1.00      0.97      6236\n",
      "          1       0.58      0.02      0.04       344\n",
      "\n",
      "avg / total       0.93      0.95      0.92      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.87      0.98      0.92      5301\n",
      "          1       0.82      0.37      0.51      1279\n",
      "\n",
      "avg / total       0.86      0.86      0.84      6580\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/sklearn/metrics/classification.py:1135: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "for i in np.arange(0,35):\n",
    "    print(classification_report(y_test.iloc[:,i], y_pred[:,i]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 8. Try improving your model further. Here are a few ideas:\n",
    "* try other machine learning algorithms\n",
    "* add other features besides the TF-IDF"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Adding genre as a feature\n",
    "\n",
    "In this part of the code, I am combining TfIdf vectorization with the binary variable representing the genre of the message. I am not planning to employ this model in the final version of the application as it does not assume to have a genre as an input. This coding part is rather done for the self-education and experimenting with pipelines.\n",
    "\n",
    "The following code is based on the following articles:\n",
    "\n",
    "https://towardsdatascience.com/custom-transformers-and-ml-data-pipelines-with-python-20ea2a7adb65\n",
    "\n",
    "https://stackoverflow.com/questions/50523930/custom-transformers-and-gridsearch-valueerror-in-pipeline\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class FeatureSelector( BaseEstimator, TransformerMixin ):\n",
    "    \"\"\"\n",
    "    This class takes the dataframe and returns only chosen feature\n",
    "    \n",
    "    Arguments:\n",
    "        X: dataframe\n",
    "\n",
    "    Returns:\n",
    "        chosen feature\n",
    "    \n",
    "    \"\"\"\n",
    "    def __init__( self, feature_names ):\n",
    "        self.feature_names = feature_names \n",
    "    \n",
    "    #Return self nothing else to do here    \n",
    "    def fit( self, X, y = None ):\n",
    "        return self \n",
    "    \n",
    "    #Method that describes what we need this transformer to do\n",
    "    def transform( self, X, y = None ):\n",
    "        \n",
    "        return X[ self.feature_names ] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LabelBinarizerPipelineFriendly(LabelBinarizer):\n",
    "    \"\"\"\n",
    "    The class binarizes a feture in a pipeline friendly way\n",
    "    \n",
    "    Arguments:\n",
    "        X: output of the class FeatureSelector\n",
    "\n",
    "    Returns:\n",
    "        binarized feature\n",
    "    \n",
    "    \"\"\"\n",
    "    def fit(self, X, y=None):\n",
    "        super(LabelBinarizerPipelineFriendly, self).fit(X)\n",
    "        \n",
    "    def transform(self, X, y=None):\n",
    "        return super(LabelBinarizerPipelineFriendly, self).transform(X)\n",
    "\n",
    "    def fit_transform(self, X, y=None):\n",
    "        return super(LabelBinarizerPipelineFriendly, self).fit(X).transform(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# combining the pipeline binarizing the variable 'genre' and the pipeline vectorizing messages\n",
    "\n",
    "binarizer_pipeline = Pipeline( steps = [ ('first_selector', FeatureSelector('genre')),\n",
    "                                         ('binarizer', LabelBinarizerPipelineFriendly()) ])\n",
    "       \n",
    "tfidf_pipeline = Pipeline( steps = [ ('second_selector', FeatureSelector('message')),\n",
    "                                     ('vect', CountVectorizer(tokenizer=tokenize)),\n",
    "                                     ('tfidf', TfidfTransformer()) ])\n",
    "\n",
    "full_pipeline = FeatureUnion( transformer_list = [ ('binarizer_pipeline', binarizer_pipeline ), \n",
    "                                                   ('tfidf_pipeline', tfidf_pipeline ) ] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# finally, applying Logistic regression to the transformed data\n",
    "full_pipeline_m = Pipeline( steps = [ ( 'full_pipeline', full_pipeline),\n",
    "                                      ( 'clf', MultiOutputClassifier(LogisticRegression(random_state=2, \n",
    "                                                                                        solver='liblinear'))) ] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {'clf__estimator__C': [0.001,0.01,0.1,1,10,100]}\n",
    "model = GridSearchCV(full_pipeline_m, params, n_jobs=-1, cv=2, verbose = 5, scoring ='roc_auc')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(memory=None,\n",
       "     steps=[('full_pipeline', FeatureUnion(n_jobs=1,\n",
       "       transformer_list=[('binarizer_pipeline', Pipeline(memory=None,\n",
       "     steps=[('first_selector', FeatureSelector(feature_names='genre')), ('binarizer', LabelBinarizerPipelineFriendly(neg_label=0, pos_label=1, sparse_output=False))])), ('tfidf_pipel...te=2, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False),\n",
       "           n_jobs=1))])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "full_pipeline_m.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 2 folds for each of 6 candidates, totalling 12 fits\n",
      "[CV] clf__estimator__C=0.001 .........................................\n",
      "[CV]  clf__estimator__C=0.001, score=0.5782246716355237, total= 1.6min\n",
      "[CV] clf__estimator__C=0.001 .........................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   1 out of   1 | elapsed:  2.4min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  clf__estimator__C=0.001, score=0.5683026728722013, total= 1.6min\n",
      "[CV] clf__estimator__C=0.01 ..........................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   2 out of   2 | elapsed:  4.8min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] . clf__estimator__C=0.01, score=0.7160395285578586, total= 1.6min\n",
      "[CV] clf__estimator__C=0.01 ..........................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   3 out of   3 | elapsed:  7.2min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] . clf__estimator__C=0.01, score=0.7068898474359514, total= 1.6min\n",
      "[CV] clf__estimator__C=0.1 ...........................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   4 out of   4 | elapsed:  9.7min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] .. clf__estimator__C=0.1, score=0.7915739258430287, total= 1.6min\n",
      "[CV] clf__estimator__C=0.1 ...........................................\n",
      "[CV] .. clf__estimator__C=0.1, score=0.7946223855935297, total= 1.6min\n",
      "[CV] clf__estimator__C=1 .............................................\n",
      "[CV] .... clf__estimator__C=1, score=0.8364084738409863, total= 1.7min\n",
      "[CV] clf__estimator__C=1 .............................................\n",
      "[CV] .... clf__estimator__C=1, score=0.8404952130397332, total= 1.6min\n",
      "[CV] clf__estimator__C=10 ............................................\n",
      "[CV] ... clf__estimator__C=10, score=0.8318678024690688, total= 1.7min\n",
      "[CV] clf__estimator__C=10 ............................................\n",
      "[CV] ... clf__estimator__C=10, score=0.8323638963249798, total= 1.7min\n",
      "[CV] clf__estimator__C=100 ...........................................\n",
      "[CV] .. clf__estimator__C=100, score=0.8148278899603496, total= 1.7min\n",
      "[CV] clf__estimator__C=100 ...........................................\n",
      "[CV] .. clf__estimator__C=100, score=0.8148920308182076, total= 1.7min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  12 out of  12 | elapsed: 29.4min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: 0.838452 using {'clf__estimator__C': 1}\n"
     ]
    }
   ],
   "source": [
    "model.fit(X_train, y_train)\n",
    "print(\"Best: %f using %s\" % (model.best_score_, model.best_params_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.91      0.97      0.94      5410\n",
      "          1       0.81      0.57      0.67      1170\n",
      "\n",
      "avg / total       0.90      0.90      0.89      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       1.00      1.00      1.00      6551\n",
      "          1       0.00      0.00      0.00        29\n",
      "\n",
      "avg / total       0.99      1.00      0.99      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.79      0.86      0.82      3844\n",
      "          1       0.77      0.69      0.73      2736\n",
      "\n",
      "avg / total       0.79      0.79      0.78      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.93      0.99      0.96      6054\n",
      "          1       0.71      0.19      0.30       526\n",
      "\n",
      "avg / total       0.92      0.93      0.91      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.96      1.00      0.98      6243\n",
      "          1       0.72      0.21      0.32       337\n",
      "\n",
      "avg / total       0.95      0.96      0.94      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.98      1.00      0.99      6424\n",
      "          1       0.91      0.06      0.12       156\n",
      "\n",
      "avg / total       0.98      0.98      0.97      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.98      1.00      0.99      6458\n",
      "          1       0.00      0.00      0.00       122\n",
      "\n",
      "avg / total       0.96      0.98      0.97      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.97      1.00      0.98      6376\n",
      "          1       0.58      0.11      0.18       204\n",
      "\n",
      "avg / total       0.96      0.97      0.96      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.97      0.99      0.98      6151\n",
      "          1       0.79      0.55      0.65       429\n",
      "\n",
      "avg / total       0.96      0.96      0.96      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.95      0.98      0.97      5832\n",
      "          1       0.84      0.63      0.72       748\n",
      "\n",
      "avg / total       0.94      0.94      0.94      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.94      0.99      0.97      5970\n",
      "          1       0.78      0.43      0.56       610\n",
      "\n",
      "avg / total       0.93      0.94      0.93      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.99      1.00      0.99      6470\n",
      "          1       0.74      0.23      0.35       110\n",
      "\n",
      "avg / total       0.98      0.99      0.98      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.98      1.00      0.99      6425\n",
      "          1       0.67      0.08      0.14       155\n",
      "\n",
      "avg / total       0.97      0.98      0.97      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.99      1.00      0.99      6511\n",
      "          1       0.00      0.00      0.00        69\n",
      "\n",
      "avg / total       0.98      0.99      0.98      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.97      1.00      0.98      6357\n",
      "          1       0.76      0.07      0.13       223\n",
      "\n",
      "avg / total       0.96      0.97      0.95      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.97      1.00      0.98      6287\n",
      "          1       0.85      0.24      0.37       293\n",
      "\n",
      "avg / total       0.96      0.96      0.95      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.88      0.99      0.93      5726\n",
      "          1       0.57      0.11      0.19       854\n",
      "\n",
      "avg / total       0.84      0.87      0.84      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.94      1.00      0.97      6153\n",
      "          1       0.33      0.02      0.03       427\n",
      "\n",
      "avg / total       0.90      0.93      0.91      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.96      1.00      0.98      6262\n",
      "          1       0.80      0.12      0.21       318\n",
      "\n",
      "avg / total       0.95      0.96      0.94      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.96      1.00      0.98      6254\n",
      "          1       0.79      0.24      0.36       326\n",
      "\n",
      "avg / total       0.95      0.96      0.95      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.98      1.00      0.99      6440\n",
      "          1       0.71      0.11      0.19       140\n",
      "\n",
      "avg / total       0.98      0.98      0.97      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.99      1.00      1.00      6538\n",
      "          1       0.00      0.00      0.00        42\n",
      "\n",
      "avg / total       0.99      0.99      0.99      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.99      1.00      1.00      6516\n",
      "          1       0.00      0.00      0.00        64\n",
      "\n",
      "avg / total       0.98      0.99      0.99      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       1.00      1.00      1.00      6553\n",
      "          1       0.00      0.00      0.00        27\n",
      "\n",
      "avg / total       0.99      1.00      0.99      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.99      1.00      0.99      6491\n",
      "          1       0.00      0.00      0.00        89\n",
      "\n",
      "avg / total       0.97      0.99      0.98      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.96      1.00      0.98      6293\n",
      "          1       0.33      0.01      0.01       287\n",
      "\n",
      "avg / total       0.93      0.96      0.94      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.88      0.95      0.91      4741\n",
      "          1       0.84      0.66      0.74      1839\n",
      "\n",
      "avg / total       0.87      0.87      0.86      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.95      1.00      0.97      6041\n",
      "          1       0.90      0.40      0.55       539\n",
      "\n",
      "avg / total       0.94      0.95      0.94      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.95      0.98      0.96      5945\n",
      "          1       0.76      0.47      0.58       635\n",
      "\n",
      "avg / total       0.93      0.93      0.93      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.99      1.00      1.00      6517\n",
      "          1       1.00      0.03      0.06        63\n",
      "\n",
      "avg / total       0.99      0.99      0.99      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.96      0.99      0.98      5955\n",
      "          1       0.90      0.65      0.75       625\n",
      "\n",
      "avg / total       0.96      0.96      0.96      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.98      1.00      0.99      6443\n",
      "          1       0.92      0.09      0.16       137\n",
      "\n",
      "avg / total       0.98      0.98      0.97      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.95      1.00      0.97      6220\n",
      "          1       0.63      0.03      0.06       360\n",
      "\n",
      "avg / total       0.93      0.95      0.92      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.88      0.96      0.92      5290\n",
      "          1       0.75      0.48      0.59      1290\n",
      "\n",
      "avg / total       0.86      0.87      0.86      6580\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/sklearn/metrics/classification.py:1135: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "for i in np.arange(0,34):\n",
    "    print(classification_report(y_test.iloc[:,i], y_pred[:,i]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Doc2vec + PCA + AdaBoost\n",
    "\n",
    "Here, I am trying to improve the results of the first classification by combining Doc2vec (instead of tf-idf), Principal Component Analysis, and Adaptive Boosting classification algorithm.\n",
    "\n",
    "The code is based on the following articles:\n",
    "\n",
    "https://www.oreilly.com/library/view/applied-text-analysis/9781491963036/ch04.html#idm140629387022352\n",
    "\n",
    "https://stackoverflow.com/questions/50278744/pipeline-and-gridsearch-for-doc2vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TextNormalizer(BaseEstimator, TransformerMixin):\n",
    "    \"\"\"\n",
    "    The class tokenizes, lowercases, and lemmatizes text, removes stopwords and punctuation\n",
    "    \n",
    "    Arguments:\n",
    "        X: pandas Series or string\n",
    "\n",
    "    Returns:\n",
    "        cleaned tokens\n",
    "    \n",
    "    \"\"\"\n",
    "    def __init__(self, language='english'):\n",
    "        self.stopwords  = set(nltk.corpus.stopwords.words(language))\n",
    "        self.lemmatizer = WordNetLemmatizer()\n",
    "    \n",
    "    def fit(self, X, y=None):\n",
    "        return self\n",
    "\n",
    "    def transform(self, documents):\n",
    "        \n",
    "        cleaned_documents = []\n",
    "        \n",
    "        for text in documents:\n",
    "            tokens = word_tokenize(text)\n",
    "            tokens = [t for t in tokens if t.lower() not in self.stopwords and t.lower() not in string.punctuation]\n",
    "        \n",
    "            clean_tokens = []\n",
    "        \n",
    "            for tok in tokens:\n",
    "                clean_tok = self.lemmatizer.lemmatize(tok).lower().strip()\n",
    "                clean_tokens.append(clean_tok)\n",
    "                cleaned_doc = ' '.join([str(elem) for elem in clean_tokens]) \n",
    "                \n",
    "            cleaned_documents.append(cleaned_doc)\n",
    "            \n",
    "        return pd.Series(cleaned_documents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Doc2VecModel(BaseEstimator):\n",
    "    '''Class transforms messages and gives an output as Doc2Vec numerical representation\n",
    "    \n",
    "    Attribute:\n",
    "        output of the class TextNormalizer\n",
    "        \n",
    "    Returns:\n",
    "        Doc2Vec numerical representation of the text\n",
    "    \n",
    "    '''\n",
    "    def __init__(self, dm=1, vector_size=300, window=2, learning_rate=0.02):\n",
    "        self.learning_rate = learning_rate\n",
    "        self.d2v_model = None\n",
    "        self.vector_size = vector_size\n",
    "        self.window = window\n",
    "        self.dm = dm\n",
    "\n",
    "    def fit(self, raw_documents, y=None):\n",
    "        # Initialize model\n",
    "        self.d2v_model = Doc2Vec(vector_size=self.vector_size, window=self.window, dm=self.dm, epochs=20, \n",
    "                                 alpha=0.025, min_alpha=0.001, learning_rate = self.learning_rate)\n",
    "        \n",
    "        # Tag docs    \n",
    "        tagged_documents = [\n",
    "            TaggedDocument(words, ['d{}'.format(idx)])\n",
    "            for idx, words in enumerate(raw_documents)\n",
    "        ]\n",
    "        \n",
    "        # Build vocabulary\n",
    "        self.d2v_model.build_vocab(tagged_documents)\n",
    "        \n",
    "        # Train model\n",
    "        self.d2v_model.train(tagged_documents, total_examples=len(tagged_documents), epochs=self.d2v_model.epochs)\n",
    "        \n",
    "        return self\n",
    "\n",
    "    def transform(self, raw_documents):\n",
    "        X = []\n",
    "        for index, row in raw_documents.iteritems():\n",
    "            X.append(self.d2v_model.infer_vector(row))\n",
    "        X = pd.DataFrame(X, index=raw_documents.index)\n",
    "        return X\n",
    "\n",
    "    def fit_transform(self, raw_documents, y=None):\n",
    "        self.fit(raw_documents)\n",
    "        return self.transform(raw_documents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipeline_doc2vec = Pipeline([\n",
    "    ('normalizer', TextNormalizer()),\n",
    "    ('doc2vec', Doc2VecModel()),\n",
    "    ('pca', PCA(n_components=100)),\n",
    "    ('clf', MultiOutputClassifier(AdaBoostClassifier(n_estimators = 200, learning_rate = 0.01)))\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(memory=None,\n",
       "     steps=[('normalizer', TextNormalizer(language=None)), ('doc2vec', Doc2VecModel(dm=1, vector_size=300, window=2)), ('pca', PCA(copy=True, iterated_power='auto', n_components=100, random_state=None,\n",
       "  svd_solver='auto', tol=0.0, whiten=False)), ('clf', MultiOutputClassifier(estimator=AdaBoostClassifier(algorithm='SAMME.R', base_estimator=None,\n",
       "          learning_rate=0.01, n_estimators=200, random_state=None),\n",
       "           n_jobs=1))])"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, Y)\n",
    "pipeline_doc2vec.fit(X_train['message'], y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = pipeline_doc2vec.predict(X_test['message'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "category_names = list(Y.columns)\n",
    "y_pred = pd.DataFrame(y_pred)\n",
    "y_pred.columns = category_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.83      1.00      0.91      5489\n",
      "          1       0.00      0.00      0.00      1091\n",
      "\n",
      "avg / total       0.70      0.83      0.76      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       1.00      1.00      1.00      6552\n",
      "          1       0.00      0.00      0.00        28\n",
      "\n",
      "avg / total       0.99      1.00      0.99      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.59      1.00      0.74      3855\n",
      "          1       0.68      0.01      0.01      2725\n",
      "\n",
      "avg / total       0.63      0.59      0.44      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.92      1.00      0.96      6081\n",
      "          1       0.00      0.00      0.00       499\n",
      "\n",
      "avg / total       0.85      0.92      0.89      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.95      1.00      0.97      6240\n",
      "          1       0.00      0.00      0.00       340\n",
      "\n",
      "avg / total       0.90      0.95      0.92      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.97      1.00      0.98      6384\n",
      "          1       0.00      0.00      0.00       196\n",
      "\n",
      "avg / total       0.94      0.97      0.96      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.98      1.00      0.99      6456\n",
      "          1       0.00      0.00      0.00       124\n",
      "\n",
      "avg / total       0.96      0.98      0.97      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.97      1.00      0.98      6358\n",
      "          1       0.00      0.00      0.00       222\n",
      "\n",
      "avg / total       0.93      0.97      0.95      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       1.00      1.00      1.00      6580\n",
      "\n",
      "avg / total       1.00      1.00      1.00      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.94      1.00      0.97      6154\n",
      "          1       0.00      0.00      0.00       426\n",
      "\n",
      "avg / total       0.87      0.94      0.90      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.89      1.00      0.94      5853\n",
      "          1       0.00      0.00      0.00       727\n",
      "\n",
      "avg / total       0.79      0.89      0.84      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.92      1.00      0.96      6048\n",
      "          1       0.00      0.00      0.00       532\n",
      "\n",
      "avg / total       0.84      0.92      0.88      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.99      1.00      0.99      6483\n",
      "          1       0.00      0.00      0.00        97\n",
      "\n",
      "avg / total       0.97      0.99      0.98      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.98      1.00      0.99      6434\n",
      "          1       0.00      0.00      0.00       146\n",
      "\n",
      "avg / total       0.96      0.98      0.97      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.99      1.00      0.99      6488\n",
      "          1       0.00      0.00      0.00        92\n",
      "\n",
      "avg / total       0.97      0.99      0.98      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.97      1.00      0.98      6353\n",
      "          1       0.00      0.00      0.00       227\n",
      "\n",
      "avg / total       0.93      0.97      0.95      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.95      1.00      0.98      6266\n",
      "          1       0.00      0.00      0.00       314\n",
      "\n",
      "avg / total       0.91      0.95      0.93      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.86      1.00      0.93      5684\n",
      "          1       0.00      0.00      0.00       896\n",
      "\n",
      "avg / total       0.75      0.86      0.80      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.94      1.00      0.97      6157\n",
      "          1       0.00      0.00      0.00       423\n",
      "\n",
      "avg / total       0.88      0.94      0.90      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.95      1.00      0.98      6266\n",
      "          1       0.00      0.00      0.00       314\n",
      "\n",
      "avg / total       0.91      0.95      0.93      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.95      1.00      0.97      6245\n",
      "          1       0.00      0.00      0.00       335\n",
      "\n",
      "avg / total       0.90      0.95      0.92      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.98      1.00      0.99      6453\n",
      "          1       0.00      0.00      0.00       127\n",
      "\n",
      "avg / total       0.96      0.98      0.97      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.99      1.00      1.00      6544\n",
      "          1       0.00      0.00      0.00        36\n",
      "\n",
      "avg / total       0.99      0.99      0.99      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.99      1.00      0.99      6511\n",
      "          1       0.00      0.00      0.00        69\n",
      "\n",
      "avg / total       0.98      0.99      0.98      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.99      1.00      1.00      6544\n",
      "          1       0.00      0.00      0.00        36\n",
      "\n",
      "avg / total       0.99      0.99      0.99      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.99      1.00      0.99      6507\n",
      "          1       0.00      0.00      0.00        73\n",
      "\n",
      "avg / total       0.98      0.99      0.98      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.96      1.00      0.98      6298\n",
      "          1       0.00      0.00      0.00       282\n",
      "\n",
      "avg / total       0.92      0.96      0.94      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.72      1.00      0.83      4708\n",
      "          1       0.50      0.00      0.00      1872\n",
      "\n",
      "avg / total       0.65      0.72      0.60      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.92      1.00      0.96      6026\n",
      "          1       0.00      0.00      0.00       554\n",
      "\n",
      "avg / total       0.84      0.92      0.88      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.91      1.00      0.95      5955\n",
      "          1       0.00      0.00      0.00       625\n",
      "\n",
      "avg / total       0.82      0.91      0.86      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.99      1.00      0.99      6513\n",
      "          1       0.00      0.00      0.00        67\n",
      "\n",
      "avg / total       0.98      0.99      0.98      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.90      1.00      0.95      5936\n",
      "          1       0.00      0.00      0.00       644\n",
      "\n",
      "avg / total       0.81      0.90      0.86      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.98      1.00      0.99      6447\n",
      "          1       0.00      0.00      0.00       133\n",
      "\n",
      "avg / total       0.96      0.98      0.97      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.95      1.00      0.97      6224\n",
      "          1       0.00      0.00      0.00       356\n",
      "\n",
      "avg / total       0.89      0.95      0.92      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.81      1.00      0.90      5338\n",
      "          1       0.00      0.00      0.00      1242\n",
      "\n",
      "avg / total       0.66      0.81      0.73      6580\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/sklearn/metrics/classification.py:1135: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "for i in category_names:\n",
    "    print(classification_report(y_test[i], y_pred[i]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The Doc2Vec model did not give a better result than the first model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Adding text statistics as features\n",
    "\n",
    "The next endeavour to improve the results assumed adding new features to the model based on the characteristics of the messages. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class text_stats(BaseEstimator, TransformerMixin):    \n",
    "    '''class calculates the length of the text and number of unique words\n",
    "    \n",
    "    Attribute:\n",
    "        string or pandas series\n",
    "        \n",
    "    Return:\n",
    "        dataframe with two columns\n",
    "    \n",
    "    '''\n",
    "    def fit(self, X, y=None):\n",
    "        return self\n",
    "\n",
    "    def transform(self, X):\n",
    "        \n",
    "        length = X.apply(len)\n",
    "        unique_words = X.apply(lambda x: len(set(w for w in x.split())))\n",
    "        \n",
    "        df = pd.concat([length, unique_words], axis=1)\n",
    "        df.columns = ['length','unique_words']\n",
    "        return df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Finally, I made two pipelines: the first one with Adaptive Boosting algorithm as a classifier, and the second one with Random Forests."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipeline_ada_ts = Pipeline([\n",
    "    ('features', FeatureUnion([\n",
    "        \n",
    "        ('nlp_pipeline', Pipeline([\n",
    "            ('vect', CountVectorizer(tokenizer=tokenize)),\n",
    "            ('tfidf', TfidfTransformer())\n",
    "        ])),\n",
    "        \n",
    "        ('text_stats', text_stats())\n",
    "    ])),\n",
    "    \n",
    "    ('clf', MultiOutputClassifier(AdaBoostClassifier()))\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(memory=None,\n",
       "     steps=[('features', FeatureUnion(n_jobs=1,\n",
       "       transformer_list=[('nlp_pipeline', Pipeline(memory=None,\n",
       "     steps=[('vect', CountVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
       "        dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
       "        lowercase=True, max_df...mator=None,\n",
       "          learning_rate=1.0, n_estimators=50, random_state=None),\n",
       "           n_jobs=1))])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipeline_ada_ts.fit(X_train['message'], y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "parameters = {'clf__estimator__n_estimators': [100, 200, 300],\n",
    "              'clf__estimator__learning_rate': [0.01, 0.1, 0.2]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "ada_ts_model = GridSearchCV(pipeline_ada_ts, param_grid=parameters, verbose = 5, n_jobs = -1, cv=2, scoring = 'roc_auc')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 2 folds for each of 9 candidates, totalling 18 fits\n",
      "[CV] clf__estimator__learning_rate=0.01, clf__estimator__n_estimators=100 \n",
      "[CV]  clf__estimator__learning_rate=0.01, clf__estimator__n_estimators=100, score=0.7360597147628853, total= 3.8min\n",
      "[CV] clf__estimator__learning_rate=0.01, clf__estimator__n_estimators=100 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   1 out of   1 | elapsed:  4.8min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  clf__estimator__learning_rate=0.01, clf__estimator__n_estimators=100, score=0.7264374500299777, total= 3.8min\n",
      "[CV] clf__estimator__learning_rate=0.01, clf__estimator__n_estimators=200 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   2 out of   2 | elapsed:  9.7min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  clf__estimator__learning_rate=0.01, clf__estimator__n_estimators=200, score=0.77044188480111, total= 6.0min\n",
      "[CV] clf__estimator__learning_rate=0.01, clf__estimator__n_estimators=200 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   3 out of   3 | elapsed: 16.8min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  clf__estimator__learning_rate=0.01, clf__estimator__n_estimators=200, score=0.77058472340617, total= 5.9min\n",
      "[CV] clf__estimator__learning_rate=0.01, clf__estimator__n_estimators=300 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   4 out of   4 | elapsed: 23.8min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  clf__estimator__learning_rate=0.01, clf__estimator__n_estimators=300, score=0.7872590202054034, total= 8.0min\n",
      "[CV] clf__estimator__learning_rate=0.01, clf__estimator__n_estimators=300 \n",
      "[CV]  clf__estimator__learning_rate=0.01, clf__estimator__n_estimators=300, score=0.784405341299615, total= 8.2min\n",
      "[CV] clf__estimator__learning_rate=0.1, clf__estimator__n_estimators=100 \n",
      "[CV]  clf__estimator__learning_rate=0.1, clf__estimator__n_estimators=100, score=0.8109587607356085, total= 3.9min\n",
      "[CV] clf__estimator__learning_rate=0.1, clf__estimator__n_estimators=100 \n",
      "[CV]  clf__estimator__learning_rate=0.1, clf__estimator__n_estimators=100, score=0.7997035700479141, total= 3.9min\n",
      "[CV] clf__estimator__learning_rate=0.1, clf__estimator__n_estimators=200 \n",
      "[CV]  clf__estimator__learning_rate=0.1, clf__estimator__n_estimators=200, score=0.8113351809565801, total= 5.9min\n",
      "[CV] clf__estimator__learning_rate=0.1, clf__estimator__n_estimators=200 \n",
      "[CV]  clf__estimator__learning_rate=0.1, clf__estimator__n_estimators=200, score=0.8040942932880606, total= 5.9min\n",
      "[CV] clf__estimator__learning_rate=0.1, clf__estimator__n_estimators=300 \n",
      "[CV]  clf__estimator__learning_rate=0.1, clf__estimator__n_estimators=300, score=0.8109609821034566, total= 8.0min\n",
      "[CV] clf__estimator__learning_rate=0.1, clf__estimator__n_estimators=300 \n",
      "[CV]  clf__estimator__learning_rate=0.1, clf__estimator__n_estimators=300, score=0.7996688567734983, total= 8.0min\n",
      "[CV] clf__estimator__learning_rate=0.2, clf__estimator__n_estimators=100 \n",
      "[CV]  clf__estimator__learning_rate=0.2, clf__estimator__n_estimators=100, score=0.8092947101492798, total= 3.9min\n",
      "[CV] clf__estimator__learning_rate=0.2, clf__estimator__n_estimators=100 \n",
      "[CV]  clf__estimator__learning_rate=0.2, clf__estimator__n_estimators=100, score=0.7988254677682005, total= 3.9min\n",
      "[CV] clf__estimator__learning_rate=0.2, clf__estimator__n_estimators=200 \n",
      "[CV]  clf__estimator__learning_rate=0.2, clf__estimator__n_estimators=200, score=0.8014591008088158, total= 6.0min\n",
      "[CV] clf__estimator__learning_rate=0.2, clf__estimator__n_estimators=200 \n",
      "[CV]  clf__estimator__learning_rate=0.2, clf__estimator__n_estimators=200, score=0.7878049942054457, total= 6.0min\n",
      "[CV] clf__estimator__learning_rate=0.2, clf__estimator__n_estimators=300 \n",
      "[CV]  clf__estimator__learning_rate=0.2, clf__estimator__n_estimators=300, score=0.7955427252433521, total= 8.1min\n",
      "[CV] clf__estimator__learning_rate=0.2, clf__estimator__n_estimators=300 \n",
      "[CV]  clf__estimator__learning_rate=0.2, clf__estimator__n_estimators=300, score=0.7836552989727359, total= 8.0min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  18 out of  18 | elapsed: 127.7min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=2, error_score='raise',\n",
       "       estimator=Pipeline(memory=None,\n",
       "     steps=[('features', FeatureUnion(n_jobs=1,\n",
       "       transformer_list=[('nlp_pipeline', Pipeline(memory=None,\n",
       "     steps=[('vect', CountVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
       "        dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
       "        lowercase=True, max_df...mator=None,\n",
       "          learning_rate=1.0, n_estimators=50, random_state=None),\n",
       "           n_jobs=1))]),\n",
       "       fit_params=None, iid=True, n_jobs=-1,\n",
       "       param_grid={'clf__estimator__n_estimators': [100, 200, 300], 'clf__estimator__learning_rate': [0.01, 0.1, 0.2]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='roc_auc', verbose=5)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ada_ts_model.fit(X_train['message'], y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: 0.807715 using {'clf__estimator__learning_rate': 0.1, 'clf__estimator__n_estimators': 200}\n"
     ]
    }
   ],
   "source": [
    "print(\"Best: %f using %s\" % (ada_ts_model.best_score_, ada_ts_model.best_params_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = ada_ts_model.predict(X_test['message'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "category_names = list(Y.columns)\n",
    "y_pred = pd.DataFrame(y_pred)\n",
    "y_pred.columns = category_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.89      0.98      0.94      5475\n",
      "          1       0.82      0.42      0.56      1105\n",
      "\n",
      "avg / total       0.88      0.89      0.87      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.99      1.00      1.00      6535\n",
      "          1       0.00      0.00      0.00        45\n",
      "\n",
      "avg / total       0.99      0.99      0.99      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.72      0.87      0.79      3851\n",
      "          1       0.75      0.53      0.62      2729\n",
      "\n",
      "avg / total       0.73      0.73      0.72      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.93      1.00      0.96      6055\n",
      "          1       0.69      0.12      0.21       525\n",
      "\n",
      "avg / total       0.91      0.93      0.90      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.95      1.00      0.98      6217\n",
      "          1       0.79      0.17      0.28       363\n",
      "\n",
      "avg / total       0.94      0.95      0.94      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.97      1.00      0.99      6385\n",
      "          1       0.67      0.10      0.18       195\n",
      "\n",
      "avg / total       0.96      0.97      0.96      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.98      1.00      0.99      6468\n",
      "          1       0.00      0.00      0.00       112\n",
      "\n",
      "avg / total       0.97      0.98      0.97      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.97      1.00      0.99      6378\n",
      "          1       0.71      0.17      0.27       202\n",
      "\n",
      "avg / total       0.97      0.97      0.96      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.97      0.99      0.98      6154\n",
      "          1       0.76      0.60      0.67       426\n",
      "\n",
      "avg / total       0.96      0.96      0.96      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.97      0.97      0.97      5838\n",
      "          1       0.79      0.75      0.77       742\n",
      "\n",
      "avg / total       0.95      0.95      0.95      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.95      0.99      0.97      5982\n",
      "          1       0.84      0.48      0.61       598\n",
      "\n",
      "avg / total       0.94      0.94      0.94      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.99      1.00      0.99      6478\n",
      "          1       0.78      0.30      0.44       102\n",
      "\n",
      "avg / total       0.99      0.99      0.99      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.98      1.00      0.99      6433\n",
      "          1       0.65      0.15      0.24       147\n",
      "\n",
      "avg / total       0.97      0.98      0.97      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.99      1.00      0.99      6500\n",
      "          1       0.71      0.12      0.21        80\n",
      "\n",
      "avg / total       0.99      0.99      0.98      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.97      1.00      0.98      6363\n",
      "          1       0.70      0.15      0.25       217\n",
      "\n",
      "avg / total       0.96      0.97      0.96      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.97      1.00      0.98      6273\n",
      "          1       0.80      0.29      0.43       307\n",
      "\n",
      "avg / total       0.96      0.96      0.96      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.88      0.99      0.93      5741\n",
      "          1       0.48      0.03      0.06       839\n",
      "\n",
      "avg / total       0.83      0.87      0.82      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.94      1.00      0.97      6157\n",
      "          1       0.60      0.02      0.04       423\n",
      "\n",
      "avg / total       0.92      0.94      0.91      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.96      1.00      0.98      6276\n",
      "          1       0.82      0.13      0.23       304\n",
      "\n",
      "avg / total       0.95      0.96      0.94      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.96      1.00      0.98      6207\n",
      "          1       0.81      0.24      0.37       373\n",
      "\n",
      "avg / total       0.95      0.95      0.94      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.98      1.00      0.99      6441\n",
      "          1       0.57      0.12      0.20       139\n",
      "\n",
      "avg / total       0.97      0.98      0.97      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.99      1.00      1.00      6542\n",
      "          1       0.00      0.00      0.00        38\n",
      "\n",
      "avg / total       0.99      0.99      0.99      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.99      1.00      0.99      6512\n",
      "          1       0.57      0.06      0.11        68\n",
      "\n",
      "avg / total       0.99      0.99      0.99      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.99      1.00      1.00      6545\n",
      "          1       0.00      0.00      0.00        35\n",
      "\n",
      "avg / total       0.99      0.99      0.99      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.99      1.00      0.99      6493\n",
      "          1       0.40      0.02      0.04        87\n",
      "\n",
      "avg / total       0.98      0.99      0.98      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.96      1.00      0.98      6304\n",
      "          1       0.67      0.01      0.03       276\n",
      "\n",
      "avg / total       0.95      0.96      0.94      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.85      0.98      0.91      4757\n",
      "          1       0.90      0.54      0.68      1823\n",
      "\n",
      "avg / total       0.86      0.86      0.84      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.96      1.00      0.98      6034\n",
      "          1       0.91      0.49      0.64       546\n",
      "\n",
      "avg / total       0.95      0.95      0.95      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.94      0.99      0.96      5970\n",
      "          1       0.82      0.37      0.51       610\n",
      "\n",
      "avg / total       0.93      0.93      0.92      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.99      1.00      0.99      6514\n",
      "          1       0.50      0.06      0.11        66\n",
      "\n",
      "avg / total       0.99      0.99      0.99      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.98      0.99      0.98      5981\n",
      "          1       0.89      0.75      0.81       599\n",
      "\n",
      "avg / total       0.97      0.97      0.97      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.98      1.00      0.99      6420\n",
      "          1       0.81      0.18      0.30       160\n",
      "\n",
      "avg / total       0.98      0.98      0.97      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.95      1.00      0.97      6225\n",
      "          1       0.63      0.03      0.06       355\n",
      "\n",
      "avg / total       0.93      0.95      0.92      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.85      0.98      0.91      5305\n",
      "          1       0.80      0.30      0.43      1275\n",
      "\n",
      "avg / total       0.84      0.85      0.82      6580\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/sklearn/metrics/classification.py:1135: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "for i in category_names:\n",
    "    print(classification_report(y_test[i], y_pred[i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipeline_rf_ts = Pipeline([\n",
    "    ('features', FeatureUnion([\n",
    "        \n",
    "        ('nlp_pipeline', Pipeline([\n",
    "            ('vect', CountVectorizer(tokenizer=tokenize)),\n",
    "            ('tfidf', TfidfTransformer())\n",
    "        ])),\n",
    "        \n",
    "        ('text_stats', text_stats())\n",
    "    ])),\n",
    "    \n",
    "    ('clf', MultiOutputClassifier(RandomForestClassifier()))\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(memory=None,\n",
       "     steps=[('features', FeatureUnion(n_jobs=1,\n",
       "       transformer_list=[('nlp_pipeline', Pipeline(memory=None,\n",
       "     steps=[('vect', CountVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
       "        dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
       "        lowercase=True, max_df...oob_score=False, random_state=None, verbose=0,\n",
       "            warm_start=False),\n",
       "           n_jobs=1))])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipeline_rf_ts.fit(X_train['message'], y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "parameters = {\n",
    "    'clf__estimator__n_estimators': [200, 300],\n",
    "    'clf__estimator__min_samples_split': [2, 3, 4]\n",
    "}\n",
    "\n",
    "rf_ts_model = GridSearchCV(pipeline_rf_ts, param_grid=parameters, verbose = 5, n_jobs = -1, cv=2, scoring = 'roc_auc')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 2 folds for each of 6 candidates, totalling 12 fits\n",
      "[CV] clf__estimator__min_samples_split=2, clf__estimator__n_estimators=200 \n",
      "[CV]  clf__estimator__min_samples_split=2, clf__estimator__n_estimators=200, score=0.8207017512042888, total= 9.7min\n",
      "[CV] clf__estimator__min_samples_split=2, clf__estimator__n_estimators=200 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   1 out of   1 | elapsed: 11.4min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  clf__estimator__min_samples_split=2, clf__estimator__n_estimators=200, score=0.8154737752523109, total= 9.8min\n",
      "[CV] clf__estimator__min_samples_split=2, clf__estimator__n_estimators=300 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   2 out of   2 | elapsed: 22.9min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  clf__estimator__min_samples_split=2, clf__estimator__n_estimators=300, score=0.8217654706562217, total=13.5min\n",
      "[CV] clf__estimator__min_samples_split=2, clf__estimator__n_estimators=300 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   3 out of   3 | elapsed: 38.5min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  clf__estimator__min_samples_split=2, clf__estimator__n_estimators=300, score=0.8205067993984909, total=13.6min\n",
      "[CV] clf__estimator__min_samples_split=3, clf__estimator__n_estimators=200 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   4 out of   4 | elapsed: 54.1min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  clf__estimator__min_samples_split=3, clf__estimator__n_estimators=200, score=0.8196061557378507, total= 8.9min\n",
      "[CV] clf__estimator__min_samples_split=3, clf__estimator__n_estimators=200 \n",
      "[CV]  clf__estimator__min_samples_split=3, clf__estimator__n_estimators=200, score=0.8195688779769639, total= 9.0min\n",
      "[CV] clf__estimator__min_samples_split=3, clf__estimator__n_estimators=300 \n",
      "[CV]  clf__estimator__min_samples_split=3, clf__estimator__n_estimators=300, score=0.8228489004068394, total=12.4min\n",
      "[CV] clf__estimator__min_samples_split=3, clf__estimator__n_estimators=300 \n",
      "[CV]  clf__estimator__min_samples_split=3, clf__estimator__n_estimators=300, score=0.8226092891949698, total=12.5min\n",
      "[CV] clf__estimator__min_samples_split=4, clf__estimator__n_estimators=200 \n",
      "[CV]  clf__estimator__min_samples_split=4, clf__estimator__n_estimators=200, score=0.8230202404436926, total= 8.6min\n",
      "[CV] clf__estimator__min_samples_split=4, clf__estimator__n_estimators=200 \n",
      "[CV]  clf__estimator__min_samples_split=4, clf__estimator__n_estimators=200, score=0.8186474131287037, total= 8.7min\n",
      "[CV] clf__estimator__min_samples_split=4, clf__estimator__n_estimators=300 \n",
      "[CV]  clf__estimator__min_samples_split=4, clf__estimator__n_estimators=300, score=0.824736326507828, total=11.7min\n",
      "[CV] clf__estimator__min_samples_split=4, clf__estimator__n_estimators=300 \n",
      "[CV]  clf__estimator__min_samples_split=4, clf__estimator__n_estimators=300, score=0.8228807106596436, total=12.1min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  12 out of  12 | elapsed: 152.5min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: 0.823809 using {'clf__estimator__min_samples_split': 4, 'clf__estimator__n_estimators': 300}\n"
     ]
    }
   ],
   "source": [
    "rf_ts_model.fit(X_train['message'], y_train)\n",
    "print(\"Best: %f using %s\" % (rf_ts_model.best_score_, rf_ts_model.best_params_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = rf_ts_model.predict(X_test['message'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "category_names = list(Y.columns)\n",
    "y_pred = pd.DataFrame(y_pred)\n",
    "y_pred.columns = category_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.91      0.98      0.94      5483\n",
      "          1       0.84      0.51      0.63      1097\n",
      "\n",
      "avg / total       0.90      0.90      0.89      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.99      1.00      1.00      6543\n",
      "          1       0.00      0.00      0.00        37\n",
      "\n",
      "avg / total       0.99      0.99      0.99      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.81      0.84      0.82      3909\n",
      "          1       0.75      0.70      0.73      2671\n",
      "\n",
      "avg / total       0.78      0.78      0.78      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.92      1.00      0.96      6050\n",
      "          1       0.67      0.05      0.10       530\n",
      "\n",
      "avg / total       0.90      0.92      0.89      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.96      1.00      0.98      6277\n",
      "          1       0.91      0.07      0.13       303\n",
      "\n",
      "avg / total       0.95      0.96      0.94      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.97      1.00      0.99      6402\n",
      "          1       1.00      0.04      0.09       178\n",
      "\n",
      "avg / total       0.97      0.97      0.96      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.98      1.00      0.99      6465\n",
      "          1       0.00      0.00      0.00       115\n",
      "\n",
      "avg / total       0.97      0.98      0.97      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.97      1.00      0.98      6366\n",
      "          1       0.69      0.05      0.10       214\n",
      "\n",
      "avg / total       0.96      0.97      0.96      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.96      1.00      0.98      6159\n",
      "          1       0.88      0.40      0.55       421\n",
      "\n",
      "avg / total       0.95      0.96      0.95      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.95      0.99      0.97      5836\n",
      "          1       0.85      0.56      0.68       744\n",
      "\n",
      "avg / total       0.94      0.94      0.93      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.95      0.99      0.97      6022\n",
      "          1       0.84      0.38      0.52       558\n",
      "\n",
      "avg / total       0.94      0.94      0.93      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.99      1.00      0.99      6481\n",
      "          1       0.75      0.03      0.06        99\n",
      "\n",
      "avg / total       0.98      0.99      0.98      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.98      1.00      0.99      6425\n",
      "          1       0.88      0.05      0.09       155\n",
      "\n",
      "avg / total       0.98      0.98      0.97      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.99      1.00      0.99      6506\n",
      "          1       1.00      0.01      0.03        74\n",
      "\n",
      "avg / total       0.99      0.99      0.98      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.97      1.00      0.98      6364\n",
      "          1       0.67      0.01      0.02       216\n",
      "\n",
      "avg / total       0.96      0.97      0.95      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.96      1.00      0.98      6303\n",
      "          1       0.83      0.14      0.24       277\n",
      "\n",
      "avg / total       0.96      0.96      0.95      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.88      1.00      0.93      5764\n",
      "          1       0.66      0.03      0.06       816\n",
      "\n",
      "avg / total       0.85      0.88      0.83      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.94      1.00      0.97      6163\n",
      "          1       0.00      0.00      0.00       417\n",
      "\n",
      "avg / total       0.88      0.94      0.91      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.96      1.00      0.98      6271\n",
      "          1       0.84      0.10      0.18       309\n",
      "\n",
      "avg / total       0.95      0.96      0.94      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.95      1.00      0.98      6234\n",
      "          1       0.88      0.11      0.19       346\n",
      "\n",
      "avg / total       0.95      0.95      0.93      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.98      1.00      0.99      6441\n",
      "          1       0.75      0.02      0.04       139\n",
      "\n",
      "avg / total       0.97      0.98      0.97      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.99      1.00      1.00      6543\n",
      "          1       0.00      0.00      0.00        37\n",
      "\n",
      "avg / total       0.99      0.99      0.99      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.99      1.00      1.00      6515\n",
      "          1       0.00      0.00      0.00        65\n",
      "\n",
      "avg / total       0.98      0.99      0.99      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       1.00      1.00      1.00      6552\n",
      "          1       0.00      0.00      0.00        28\n",
      "\n",
      "avg / total       0.99      1.00      0.99      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.99      1.00      1.00      6515\n",
      "          1       0.00      0.00      0.00        65\n",
      "\n",
      "avg / total       0.98      0.99      0.99      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.96      1.00      0.98      6288\n",
      "          1       0.00      0.00      0.00       292\n",
      "\n",
      "avg / total       0.91      0.96      0.93      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.89      0.95      0.92      4792\n",
      "          1       0.84      0.70      0.76      1788\n",
      "\n",
      "avg / total       0.88      0.88      0.88      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.95      1.00      0.98      6033\n",
      "          1       0.92      0.48      0.63       547\n",
      "\n",
      "avg / total       0.95      0.95      0.95      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.95      0.98      0.97      5971\n",
      "          1       0.78      0.52      0.63       609\n",
      "\n",
      "avg / total       0.94      0.94      0.94      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.99      1.00      0.99      6496\n",
      "          1       0.00      0.00      0.00        84\n",
      "\n",
      "avg / total       0.97      0.99      0.98      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.98      0.99      0.98      6017\n",
      "          1       0.88      0.78      0.83       563\n",
      "\n",
      "avg / total       0.97      0.97      0.97      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.98      1.00      0.99      6464\n",
      "          1       0.77      0.09      0.16       116\n",
      "\n",
      "avg / total       0.98      0.98      0.98      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.95      1.00      0.97      6238\n",
      "          1       0.73      0.02      0.05       342\n",
      "\n",
      "avg / total       0.94      0.95      0.93      6580\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.87      0.98      0.92      5321\n",
      "          1       0.82      0.38      0.52      1259\n",
      "\n",
      "avg / total       0.86      0.87      0.85      6580\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/sklearn/metrics/classification.py:1135: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "for i in category_names:\n",
    "    print(classification_report(y_test[i], y_pred[i]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Random Forest classifier gave the best result and was chosen as a final model for the application."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 9. Export your model as a pickle file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "pkl_filename = \"rf_ts_model.pkl\"\n",
    "with open(pkl_filename, 'wb') as file:\n",
    "    pickle.dump(rf_ts_model, file)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}