Features.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from nltk.corpus import stopwords\n",
    "import nltk, os, re, string, collections\n",
    "import datetime as dt\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from  imageio import imread\n",
    "from nltk.util import ngrams\n",
    "from nltk.corpus import stopwords\n",
    "import string\n",
    "import multidict as multidict #pip install multidict\n",
    "import itertools\n",
    "#import csvtomd\n",
    "import re\n",
    "from collections import Counter\n",
    "#%matplotlib inline\n",
    "import spacy\n",
    "from sklearn.preprocessing import MultiLabelBinarizer, Binarizer\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Limpieza y conversión de datos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "dtype_test = {\"id\": np.int32, \"keyword\": \"category\"}\n",
    "test = pd.read_csv(\"original_data/test.csv\", dtype = dtype_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "dtype_train = {\"id\": np.int32, \"keyword\": \"category\", \"target\" : int}\n",
    "train = pd.read_csv(\"original_data/train.csv\", dtype = dtype_train, encoding='UTF_8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "target_train = train.target"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.1 Keywords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp = spacy.load('en', disable=['parser', 'ner'])\n",
    "\n",
    "def lemmatize_text(text):\n",
    "    w_tokenizer = nltk.tokenize.WhitespaceTokenizer()\n",
    "    lemmatizer  = nltk.stem.WordNetLemmatizer()\n",
    "    if text == '' or pd.isnull(text):\n",
    "        return text\n",
    "    else:\n",
    "        wordList = [lemmatizer.lemmatize(w) for w in w_tokenizer.tokenize(text)]\n",
    "        s = ' '\n",
    "        return s.join(wordList)\n",
    "    \n",
    "def lemmatize_text_v2(text):\n",
    "    if text == '' or pd.isnull(text):\n",
    "        return text\n",
    "    else:\n",
    "        doc = nlp(text)\n",
    "        return \" \".join([token.lemma_ for token in doc])\n",
    "\n",
    "def clean_keyword_field(df):\n",
    "    df[\"keyword\"] = df[\"keyword\"].str.replace('%20',' ')\n",
    "    df_sin_keywords_nulos = df.dropna(subset = ['keyword'])\n",
    "    keywords = np.unique(df_sin_keywords_nulos.keyword)\n",
    "    for keyword in keywords:\n",
    "        df.loc[(df['text'].str.contains(keyword, case=False) & df['keyword'].isnull()), 'keyword'] = keyword\n",
    "    #df['keyword'] = df['keyword'].apply(lemmatize_text_v2)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.2 Text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Strict (for tags)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2253\n"
     ]
    }
   ],
   "source": [
    "def getStopwordsList():\n",
    "    fileNamesList = [\"texts/99webtools.txt\", \"texts/atire_ncbi.txt\", \"texts/atire_puurula.txt\", \"texts/azure.txt\", \n",
    "                 \"texts/bbalet.txt\", \n",
    "                 \"texts/bow_short.txt\", \"texts/choi_2000naacl.txt\", \"texts/cook1988_function_words.txt\", \n",
    "                 \"texts/corenlp_acronym.txt\", \n",
    "                 \"texts/corenlp_hardcoded.txt\", \"texts/corenlp_stopwords.txt\", \"texts/datasciencedojo.txt\", \n",
    "                 \"texts/deeplearning4j.txt\", \n",
    "                 \"texts/dkpro.txt\", \"texts/mongodb.txt\", \"texts/galago_inquery.txt\", \"texts/gate_keyphrase.txt\", \n",
    "                 \"texts/gensim.txt\", \n",
    "                 \"texts/glasgow_stop_words.txt\", \"texts/indri.txt\", \"texts/kevinbouge.txt\", \"texts/lexisnexis.txt\",\n",
    "                 \"texts/lingpipe.txt\", \n",
    "                 \"texts/mallet.txt\", \"texts/mysql_innodb.txt\", \"texts/mysql_myisam.txt\", \"texts/galago_rmstop.txt\", \n",
    "                 \"texts/atire_ncbi.txt\", \n",
    "                 \"texts/galago_rmstop.txt\", \"texts/nltk.txt\", \"texts/okapiframework.txt\", \"texts/okapi_cacm_expanded.txt\", \n",
    "                 \"texts/onix.txt\", \n",
    "                 \"texts/ovid.txt\", \"texts/postgresql.txt\", \"texts/pubmed.txt\", \"texts/quanteda.txt\", \"texts/r_tm.txt\", \n",
    "                 \"texts/ranksnl_large.txt\", \n",
    "                 \"texts/reuters_wos.txt\", \"texts/rouge_155.txt\", \"texts/scikitlearn.txt\", \"texts/smart.txt\", \n",
    "                 \"texts/snowball_expanded.txt\", \n",
    "                 \"texts/spacy.txt\", \"texts/spark_mllib.txt\", \"texts/sphinx_mirasvit.txt\", \"texts/t101_minimal.txt\", \n",
    "                 \"texts/taporware.txt\", \n",
    "                 \"texts/terrier.txt\", \"texts/tonybsk_1.txt\", \"texts/tonybsk_6.txt\", \"texts/voyant_taporware.txt\", \n",
    "                 \"texts/weka.txt\", \n",
    "                 \"texts/xapian.txt\", \"texts/xpo6.txt\", \"texts/zettair.txt\"]\n",
    "    stopwordsList = []\n",
    "    for fileName in fileNamesList:\n",
    "        file = open(fileName, \"r\", encoding=\"utf8\")\n",
    "        for line in file:\n",
    "            stripped_line = line. strip()\n",
    "            line_list = stripped_line\n",
    "            if line_list not in stopwordsList:\n",
    "                stopwordsList.append(line_list)\n",
    "        file.close()\n",
    "    return stopwordsList\n",
    "\n",
    "stopwordsList = getStopwordsList()\n",
    "\n",
    "print(len(stopwordsList))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_text_strict(text):\n",
    "    text = text.lower()\n",
    "    text = text.replace('\\d+', '')\n",
    "    invalid_chars = ['#','|','@','!','?','-','_','[',']','%','&',':','.',',',\"''\",'/','https','(','//t',')','http',\n",
    "                 ';','\\'']\n",
    "    for char in invalid_chars:\n",
    "        if char in text:\n",
    "            text = text.replace(char,' ')\n",
    "    #removes url and tags\n",
    "    text = re.sub(r\"(?:\\@|https?\\://)\\S+\", \" \",  text)\n",
    "    return text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Non strict (for special characters, hashtags, etc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_text_non_strict(text): \n",
    "    tw = \" \"\n",
    "    words = text.split(' ')\n",
    "    for word in words:\n",
    "        word = ''.join(filter(lambda x: x in set(string.printable), word))\n",
    "        word = word.replace(\"\\n\",\" \")\n",
    "        word = word.replace('û',\"\")\n",
    "        word = word.replace('Û',\"\")\n",
    "        #word = word.replace('_','')\n",
    "        #word = word.replace(\"\\\"\",'')\n",
    "        #word = word.strip('.')\n",
    "        #word = word.strip(',')\n",
    "        #word = word.strip(':')\n",
    "        tw += word + \" \"\n",
    "    return tw"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.3 Location"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#FUNCIONES AUXILIARES: Limpieza de tweet\n",
    "\n",
    "# Función que saca los links\n",
    "\n",
    "def quitar_link_twitter(tweet):\n",
    "    res = []\n",
    "    for w in tweet.split(\" \"):\n",
    "        if ((\"http\" not in w) and (\"https\" not in w)):\n",
    "            res.append(w)\n",
    "    return (\" \").join(res)\n",
    "\n",
    "# Función que saca las menciones\n",
    "\n",
    "def quitar_mencion_twitter(tweet):\n",
    "    res = []\n",
    "    for w in tweet.split(\" \"):\n",
    "        if(\"@\" not in w):\n",
    "            res.append(w)\n",
    "    return (\" \").join(res)\n",
    "\n",
    "def quitar_simbolo(tweet, simbolo):\n",
    "    text = \"\"\n",
    "    for c in tweet:\n",
    "        if(c!=simbolo):\n",
    "            text+=c\n",
    "    return text\n",
    "\n",
    "def agregar_espacio_ente_comas(tweet):\n",
    "    \n",
    "    tweer_sin_coma = (\" \").join(tweet.split(\",\"))\n",
    "\n",
    "    return (\" \").join(tweer_sin_coma.split(\".\"))\n",
    "\n",
    "def capitalize_each_word(string):\n",
    "    \n",
    "    text = \"\"\n",
    "    for w in string.split(\" \"):\n",
    "        if(len(w)>3):\n",
    "            text += w.capitalize()+\" \"\n",
    "    return text\n",
    "\n",
    "def limpiar_tweet_location(tweet):\n",
    "    res = tweet\n",
    "    func = [quitar_mencion_twitter, quitar_link_twitter, agregar_espacio_ente_comas, lambda x: quitar_simbolo(x, \"#\"), capitalize_each_word]\n",
    "    for f in func:\n",
    "        res = f(res)\n",
    "    return res\n",
    "\n",
    "def to_string(a_list, delimeter):\n",
    "    res = \"\"\n",
    "    \n",
    "    for e in sorted(filter(None, a_list)):\n",
    "        res += str(e)+delimeter\n",
    "    \n",
    "    if(len(res)==0): return None\n",
    "    return res[: -len(delimeter)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# FUNCIONES AUXILIARES: Analisis geografico\n",
    "\n",
    "from geotext import GeoText #pip install https://github.com/elyase/geotext/archive/master.zip\n",
    "import pycountry\n",
    "import geonamescache # pip install geonamescache\n",
    "\n",
    "gc = geonamescache.GeonamesCache()\n",
    "\n",
    "def get_country_pycountry(text):\n",
    "    for country in pycountry.countries:\n",
    "        # Handle both the cases(Uppercase/Lowercase)\n",
    "        if str(country.name).lower() in str(text).lower():\n",
    "            return GeoText(country.name).country_mentions\n",
    "\n",
    "def get_country_geotext(text):\n",
    "    return GeoText(text).country_mentions\n",
    "\n",
    "def get_country(text):\n",
    "    set1 = set(get_country_geotext(text))\n",
    "    countries = get_country_pycountry(text)\n",
    "    if(not countries):\n",
    "        return list(set1)\n",
    "    return list(set1.union(set(countries)))\n",
    "\n",
    "def check_if_US_state(text):\n",
    "    stateList = []\n",
    "    states = gc.get_us_states_by_names()\n",
    "    for state in states:\n",
    "        if (state in text):\n",
    "            stateList.append(state)\n",
    "    return list(set(stateList))\n",
    "\n",
    "def check_if_US_county(text):\n",
    "    countyList = []\n",
    "    stateList = []\n",
    "    counties = gc.get_us_counties()\n",
    "    for county in counties:\n",
    "        name = county.get('name')\n",
    "        if name in text:\n",
    "            countyList.append(name)\n",
    "            stateList.append(county.get('state'))\n",
    "    return list(set(countyList))\n",
    "\n",
    "def check_if_continent(text):\n",
    "    continentList = []\n",
    "    continents = gc.get_continents()\n",
    "    for continent,info in continents.items():\n",
    "        if (info.get('asciiName') in text):\n",
    "            continentList.append(continent)\n",
    "    return continentList\n",
    "\n",
    "def get_US_state(county):\n",
    "    stateList = []\n",
    "    counties = gc.get_us_counties()\n",
    "    if (pd.isnull(county)):\n",
    "        return county\n",
    "    for c in counties:\n",
    "        name = c.get('name')\n",
    "        if name in county:\n",
    "            stateList.append(c.get('state'))\n",
    "    return list(set(stateList))\n",
    "\n",
    "def get_continent(country):\n",
    "    continentList = []\n",
    "    countries = gc.get_countries()\n",
    "    if (pd.isnull(country)):\n",
    "        return continentList\n",
    "    country = country.split(',')\n",
    "    for c in country:\n",
    "        info = countries.get(c)\n",
    "        continentList.append(info.get('continentcode'))\n",
    "    return list(set(continentList))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Location in text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creo dataframe y limpio la location\n",
    "\n",
    "#location_in_text = train[[\"id\", \"text\"]].copy()\n",
    "#location_in_text['text'] = location_in_text['text'].apply(lambda x: limpiar_tweet_location(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Creo columnas con informacion\n",
    "#location_in_text[\"cities\"] = location_in_text[\"text\"].apply(lambda x: GeoText(str(x)).cities)\n",
    "#location_in_text[\"cities_str\"] = location_in_text[\"cities\"].apply(lambda x: to_string(x,\",\"))\n",
    "\n",
    "#location_in_text[\"country\"] = location_in_text[\"text\"].apply(lambda x: get_country(x))\n",
    "\n",
    "#location_in_text[\"counties\"] = location_in_text[\"text\"].apply(lambda x: check_if_US_county(str(x)))\n",
    "#location_in_text[\"counties_str\"] = location_in_text[\"counties\"].apply(lambda x: to_string(x,\",\"))\n",
    "\n",
    "#location_in_text[\"states\"] = location_in_text[\"text\"].apply(lambda x: check_if_US_state(str(x)))\n",
    "#location_in_text.loc[(location_in_text['states'].str.len() == 0), 'states'] = location_in_text.counties.apply(lambda x: get_US_state(x))\n",
    "\n",
    "#location_in_text[\"country_str\"] = location_in_text[\"country\"].apply(lambda x: to_string(x,\",\"))\n",
    "#location_in_text[\"states_str\"] = location_in_text[\"states\"].apply(lambda x: to_string(x,\",\"))\n",
    "#location_in_text.loc[~pd.isnull(location_in_text['counties_str']), 'country_str'] = 'US'\n",
    "#location_in_text.loc[~pd.isnull(location_in_text['states_str']), 'country_str'] = 'US'\n",
    "\n",
    "#location_in_text[\"continent\"] = location_in_text[\"text\"].apply(lambda x: check_if_continent(str(x)))\n",
    "#location_in_text.loc[location_in_text['continent'].str.len() == 0, 'continent'] = location_in_text.country_str.apply(lambda x: get_continent(x))\n",
    "\n",
    "#location_in_text[\"continent_str\"] = location_in_text[\"continent\"].apply(lambda x: to_string(x,\",\"))\n",
    "\n",
    "#location_in_text[\"has_country\"] = np.where(pd.isnull(location_in_text['country_str']), False, True)\n",
    "\n",
    "#location_in_text.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Selecciono las columnas más útiles para crear features\n",
    "\n",
    "#location_in_text = location_in_text[[\"id\",\"counties_str\",\"cities_str\",\"states_str\",\"country_str\",\"continent_str\",\"has_country\"]]\n",
    "#location_in_text.columns = [\"id\", \"has_location\", \"cities\", \"country\"]\n",
    "#location_in_text.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#location_in_text.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Location label info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creo dataframe y limpio la location\n",
    "\n",
    "#location = train[[\"id\", \"location\",\"target\"]].copy().fillna(\"\")\n",
    "#location[\"location\"] = location[\"location\"].apply(lambda x: limpiar_tweet_location(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Creo columnas con informacion\n",
    "\n",
    "#location[\"has_location\"] = location[\"location\"]!=\"\"\n",
    "\n",
    "#location[\"cities\"] = location[\"location\"].apply(lambda x: GeoText(str(x)).cities)\n",
    "#location[\"cities_str\"] = location[\"cities\"].apply(lambda x: to_string(x,\",\"))\n",
    "\n",
    "#location[\"country\"] = location[\"location\"].apply(lambda x: get_country(x))\n",
    "#location[\"country_str\"] = location[\"country\"].apply(lambda x: to_string(x,\",\"))\n",
    "\n",
    "#location[\"cities\"] = location[\"location\"].apply(lambda x: GeoText(str(x)).cities)\n",
    "#location[\"cities_str\"] = location[\"cities\"].apply(lambda x: to_string(x,\",\"))\n",
    "\n",
    "#location[\"country\"] = location[\"location\"].apply(lambda x: get_country(x))\n",
    "\n",
    "#location[\"counties\"] = location[\"location\"].apply(lambda x: check_if_US_county(str(x)))\n",
    "#location[\"counties_str\"] = location[\"counties\"].apply(lambda x: to_string(x,\",\"))\n",
    "\n",
    "#location[\"states\"] = location[\"location\"].apply(lambda x: check_if_US_state(str(x)))\n",
    "#location.loc[(location['states'].str.len() == 0), 'states'] = location.counties.apply(lambda x: get_US_state(x))\n",
    "\n",
    "#location[\"country_str\"] = location[\"country\"].apply(lambda x: to_string(x,\",\"))\n",
    "#location[\"states_str\"] = location[\"states\"].apply(lambda x: to_string(x,\",\"))\n",
    "#location.loc[~pd.isnull(location['counties_str']), 'country_str'] = 'US'\n",
    "#location.loc[~pd.isnull(location['states_str']), 'country_str'] = 'US'\n",
    "\n",
    "#location[\"continent\"] = location[\"location\"].apply(lambda x: check_if_continent(str(x)))\n",
    "#location.loc[location['continent'].str.len() == 0, 'continent'] = location.country_str.apply(lambda x: get_continent(x))\n",
    "\n",
    "#location[\"continent_str\"] = location[\"continent\"].apply(lambda x: to_string(x,\",\"))\n",
    "\n",
    "#location[\"has_country\"] = np.where(pd.isnull(location['country_str']), False, True)\n",
    "\n",
    "#location[location['has_location']==True].head(50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "#location_aux = location[[\"country_str\", \"cities_str\", \"states_str\", \"continent_str\", \"counties_str\"]].isnull()[520:525]\n",
    "#location[\"city_country_not_found\"] = location_aux[\"country_str\"] & location_aux[\"cities_str\"] & location_aux[\"counties_str\"] & location_aux[\"continent_str\"] & location_aux[\"states_str\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Selecciono las columnas más útiles para crear features\n",
    "\n",
    "#location_info = location[[\"id\", \"has_location\", \"cities_str\", \"country_str\", \"city_country_not_found\", \"counties_str\", \"states_str\", \"continent_str\"]]\n",
    "#location_info.columns = [\"id\", \"has_location\", \"cities\", \"country\", \"city_country_not_found\", \"county\", \"state\", \"continent\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "#location_info.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.4 Links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "links_train = pd.read_csv(\"new_data/links_train.csv\", encoding='UTF_8').set_index(\"id\")\n",
    "links_test = pd.read_csv(\"new_data/links_test.csv\", encoding='UTF_8').set_index(\"id\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import ast\n",
    "\n",
    "links_train['http']= links_train['http'].apply(lambda x: ast.literal_eval(x))\n",
    "links_test['http']= links_test['http'].apply(lambda x: ast.literal_eval(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Funciones auxiliares\n",
    "\n",
    "def get_dominio(links):\n",
    "    res= []\n",
    "    for l in links:\n",
    "        if(l==None):\n",
    "            res.append(None)\n",
    "        else:\n",
    "            dom = l.split(\"/\")\n",
    "            res.append(dom[2])\n",
    "    return res\n",
    "\n",
    "def type_http(links):\n",
    "    res = []\n",
    "    for l in links:\n",
    "        if(l==None):\n",
    "            res.append(None)\n",
    "        else:\n",
    "            dom = l.split(\":\")\n",
    "            res.append(dom[0])\n",
    "    return res\n",
    "\n",
    "def count_None(a_list):\n",
    "    count = 0\n",
    "    for l in a_list:\n",
    "        if(l==None):\n",
    "            count+=1\n",
    "    return count\n",
    "\n",
    "def link_encription_generalize(link_encript):\n",
    "    \n",
    "    has_http = False\n",
    "    has_https = False\n",
    "    \n",
    "    for type_http in link_encript:\n",
    "        if type_http==None: \n",
    "            continue\n",
    "        elif type_http== \"https\":\n",
    "            has_https = True\n",
    "        else:\n",
    "            has_http = True\n",
    "    \n",
    "    if(has_http and has_https):\n",
    "        return \"both\"\n",
    "    elif(has_http):\n",
    "        return \"http\"\n",
    "    elif(has_https):\n",
    "        return \"https\"\n",
    "    else:\n",
    "        return None\n",
    "    \n",
    "def to_string(a_list, delimeter):\n",
    "    res = \"\"\n",
    "    \n",
    "    for e in sorted(filter(None, a_list)):\n",
    "        res += str(e)+delimeter\n",
    "    \n",
    "    if(len(res)==0): return None\n",
    "    return res[: -len(delimeter)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.5 Tags"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Limpieza de los campos para obtener tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# FUNCIONES AUXILIARES\n",
    "\n",
    "def prettify(word_list):\n",
    "    wl = []\n",
    "    for word in word_list:\n",
    "        word = re.sub(r'[^A-Za-z]', \" \",  word)\n",
    "        word = re.sub(r'\\b\\w\\b',' ', word)\n",
    "        word = re.sub(r'\\s+', ' ', word)\n",
    "        wl.append(word.strip())\n",
    "    return wl\n",
    "\n",
    "def lemmatize_text2(word_list):\n",
    "    text = ' '\n",
    "    text = text.join(word_list)\n",
    "    return [lemmatizer.lemmatize(w) for w in w_tokenizer.tokenize(text)]\n",
    "\n",
    "def getText(word_list):\n",
    "    text = ' '\n",
    "    text = text.join(word_list)\n",
    "    return text\n",
    "\n",
    "def word_finder(x):\n",
    "    x = set(x)\n",
    "    return list(unique_frequent_words.intersection(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creo el dataframe\n",
    "#df_tags = train[['id','text','target']].copy(deep=True)\n",
    "\n",
    "# Limpio el dataframe\n",
    "#df_tags['new'] = df_tags[\"text\"].apply(lambda x: clean_text_strict(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "#  Eliminacion de terminos, caracteres y stopwords innecesarios para generar los tags\n",
    "#convierte de list a string\n",
    "\n",
    "#df_tags['new'] = df_tags['new'].apply(lambda x: [item for item in x.split() if item not in stopwordsList])\n",
    "#df_tags.new = df_tags.new.apply(prettify)\n",
    "\n",
    "#df_tags.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tokenize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "#w_tokenizer = nltk.tokenize.WhitespaceTokenizer()\n",
    "#lemmatizer = nltk.stem.WordNetLemmatizer()\n",
    "\n",
    "#df_tags['new_tokenized'] = df_tags.new.apply(lemmatize_text2)\n",
    "#df_tags.new_tokenized = df_tags.new_tokenized.apply(prettify)\n",
    "\n",
    "#df_tags.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Analisis de ocurrencias"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "#metodo para analizar ocurrencias y cantidades\n",
    "#a = df_tags['new_tokenized'].apply(getText).str.lower().str.cat(sep=' ')\n",
    "\n",
    "#words = nltk.tokenize.word_tokenize(a)\n",
    "#word_dist = nltk.FreqDist(words)\n",
    "\n",
    "#this_col = pd.Series(['word','frequency'])\n",
    "\n",
    "#rslt = pd.DataFrame(word_dist.most_common(400),columns = this_col)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "#frequent_words = rslt ['word']\n",
    "#unique_frequent_words = set(frequent_words)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Se genera columna con tags a partir de una cantidad de ocurrencias destacadas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.1 Keywords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_keywords = train.copy()\n",
    "train_keywords = clean_keyword_field(train_keywords)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_keywords_lemmatized = train_keywords.copy()\n",
    "train_keywords_lemmatized.keyword = train_keywords.keyword.apply(lambda x: lemmatize_text_v2(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_just_links = links_train.reset_index()[['http','id']]\n",
    "train_keywords_links = train_keywords.merge(train_just_links, on='id', how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_keywords = test.copy()\n",
    "test_keywords = clean_keyword_field(test_keywords)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_keywords_lemmatized = test_keywords.copy()\n",
    "test_keywords_lemmatized.keyword = test_keywords.keyword.apply(lambda x: lemmatize_text_v2(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_just_links = links_test.reset_index()[['http','id']]\n",
    "test_keywords_links = test_keywords.merge(test_just_links, on='id', how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getKeywords(df):\n",
    "    df_sin_keywords_nulos = df.dropna(subset = ['keyword'])\n",
    "    return np.unique(df_sin_keywords_nulos.keyword)\n",
    "\n",
    "keywords_lemmatized_train = getKeywords(train_keywords_lemmatized)\n",
    "keywords_train = getKeywords(train_keywords)\n",
    "keywords_lemmatized_test = getKeywords(test_keywords_lemmatized)\n",
    "keywords_test = getKeywords(test_keywords)\n",
    "\n",
    "def countKeywords(tweet, train):\n",
    "    if (train == True):\n",
    "        keywords = keywords_train\n",
    "    else:\n",
    "        keywords = keywords_test\n",
    "    quantity = 0\n",
    "    tweet = re.sub('[^a-zA-Z\\ ]+', \"\", tweet).lower()\n",
    "    for word in tweet.split(' '):\n",
    "        #word = re.sub('[^a-zA-Z]+', '', word)\n",
    "        if word in keywords:\n",
    "            quantity+=1\n",
    "    return quantity\n",
    "\n",
    "def getHiddenKeywords(tweet, train):\n",
    "    if (train == True):\n",
    "        keywords = keywords_train\n",
    "    else:\n",
    "        keywords = keywords_test\n",
    "    k = []\n",
    "    tweet = re.sub('[^a-zA-Z\\ ]+', \"\", tweet).lower()\n",
    "    for keyword in keywords:\n",
    "        if keyword in tweet:\n",
    "            k.append(lemmatize_text_v2(keyword))\n",
    "    return k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "def toText(listOfWords):\n",
    "    s=' '\n",
    "    return s.join(listOfWords)\n",
    "\n",
    "def textContainsKeyword(row):\n",
    "    if row.keyword == '' or pd.isnull(row.keyword):\n",
    "        return False\n",
    "    else:\n",
    "        return row.keyword.lower() in row.text.lower()\n",
    "\n",
    "def textContainsKeywordAsHashtag(row):\n",
    "    hashtags = []\n",
    "    tweet = row.text.lower()\n",
    "    if row.keyword == '' or pd.isnull(row.keyword):\n",
    "        return False\n",
    "    else:\n",
    "        for word in tweet.split(' '):\n",
    "            if (len(word)>1) and (word[0] == '#'):\n",
    "                hashtags.append(word[1:])\n",
    "        return row.keyword.lower() in hashtags\n",
    "\n",
    "def linkContainsKeyword(row):\n",
    "    links   = row.http\n",
    "    keyword = row.keyword\n",
    "    if row.keyword == '' or pd.isnull(row.keyword):\n",
    "        return False\n",
    "    if (type(links)==list):\n",
    "        for link in links:\n",
    "            if link=='' or pd.isnull(link):\n",
    "                continue\n",
    "            if keyword in link:\n",
    "                return True\n",
    "    else:\n",
    "        return False\n",
    "    return False\n",
    "\n",
    "def keywords_agregar_features_numericos(df, train):\n",
    "    if (train == True):\n",
    "        df_keywords_lemmatized = train_keywords_lemmatized\n",
    "        keywords_lemmatized    = keywords_lemmatized_train\n",
    "        keywords               = keywords_train\n",
    "    else:\n",
    "        df_keywords_lemmatized = test_keywords_lemmatized\n",
    "        keywords_lemmatized    = keywords_lemmatized_test\n",
    "        keywords               = keywords_test\n",
    "    df['text_contains_keyword'] = (df.apply(textContainsKeyword,axis=1)).astype(int)\n",
    "    df['has_keyword'] = (~df.keyword.isnull()).astype(int)\n",
    "    df['keywords_quantity'] = df.text.apply(countKeywords, args=(train,))\n",
    "    df['keywords_mean'] = df.text.apply(countKeywords, args=(train,))/keywords.size\n",
    "    df['keyword_is_hashtag'] = (df.apply(textContainsKeywordAsHashtag,axis=1)).astype(int)\n",
    "    df['keyword_frequency'] = df_keywords_lemmatized.groupby('keyword')['id'].transform('count') / keywords_lemmatized.size\n",
    "#    df['link_contains_keyword'] = (df_keywords_links.apply(linkContainsKeyword,axis=1)).astype(int)\n",
    "    return df\n",
    "\n",
    "def keywords_agregar_features_categoricos(df, train):\n",
    "    df['keywords_in_text'] = df.text.apply(getHiddenKeywords, args=(train,))\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "#all_keywords = pd.get_dummies(df_keywords_lemmatized[['id','keyword']],'keyword')\n",
    "#all_keywords.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_keywords = df_train.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_sin_keywords_nulos = df_keywords.dropna(subset=['keyword'])\n",
    "#%time df_keywords_not_in_text = df_sin_keywords_nulos[~df_sin_keywords_nulos['text'].str.contains(\"|\".join(df_sin_keywords_nulos['keyword']),case=False)]\n",
    "#%time df_sin_keywords_nulos[df_sin_keywords_nulos.apply(lambda x: x.keyword in x.text, axis=1)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>target</th>\n",
       "      <th>text_contains_keyword</th>\n",
       "      <th>has_keyword</th>\n",
       "      <th>keywords_quantity</th>\n",
       "      <th>keywords_mean</th>\n",
       "      <th>keyword_is_hashtag</th>\n",
       "      <th>keyword_frequency</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.004525</td>\n",
       "      <td>1</td>\n",
       "      <td>0.227027</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.004525</td>\n",
       "      <td>0</td>\n",
       "      <td>0.237838</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.004525</td>\n",
       "      <td>0</td>\n",
       "      <td>0.210811</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.004525</td>\n",
       "      <td>0</td>\n",
       "      <td>0.210811</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.004525</td>\n",
       "      <td>0</td>\n",
       "      <td>0.237838</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  target  text_contains_keyword  has_keyword  keywords_quantity  \\\n",
       "0   1       1                      1            1                  1   \n",
       "1   4       1                      1            1                  1   \n",
       "2   5       1                      1            1                  1   \n",
       "3   6       1                      1            1                  1   \n",
       "4   7       1                      1            1                  1   \n",
       "\n",
       "   keywords_mean  keyword_is_hashtag  keyword_frequency  \n",
       "0       0.004525                   1           0.227027  \n",
       "1       0.004525                   0           0.237838  \n",
       "2       0.004525                   0           0.210811  \n",
       "3       0.004525                   0           0.210811  \n",
       "4       0.004525                   0           0.237838  "
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "keywords_numerical_features = train_keywords.copy()\n",
    "keywords_numerical_features = keywords_agregar_features_numericos(keywords_numerical_features, True).drop(columns=['location','text','keyword'])\n",
    "keywords_numerical_features.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Encoding de variables categoricas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>target</th>\n",
       "      <th>keywords_in_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>[earthquake]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>[fire, forest fire]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>[evacuation]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>[evacuation, fire, wildfire]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>[fire, smoke, wildfire]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  target              keywords_in_text\n",
       "0   1       1                  [earthquake]\n",
       "1   4       1           [fire, forest fire]\n",
       "2   5       1                  [evacuation]\n",
       "3   6       1  [evacuation, fire, wildfire]\n",
       "4   7       1       [fire, smoke, wildfire]"
      ]
     },
     "execution_count": 219,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "keywords_categorical_features = train_keywords.copy() \n",
    "keywords_categorical_features = keywords_agregar_features_categoricos(keywords_categorical_features,True).drop(columns=['keyword','location','text'])\n",
    "keywords_categorical_features.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>target</th>\n",
       "      <th>K_ablaze</th>\n",
       "      <th>K_accident</th>\n",
       "      <th>K_aftershock</th>\n",
       "      <th>K_airplane accident</th>\n",
       "      <th>K_ambulance</th>\n",
       "      <th>K_annihilate</th>\n",
       "      <th>K_annihilation</th>\n",
       "      <th>K_apocalypse</th>\n",
       "      <th>...</th>\n",
       "      <th>K_volcano</th>\n",
       "      <th>K_war zone</th>\n",
       "      <th>K_weapon</th>\n",
       "      <th>K_whirlwind</th>\n",
       "      <th>K_wild fire</th>\n",
       "      <th>K_wildfire</th>\n",
       "      <th>K_windstorm</th>\n",
       "      <th>K_wound</th>\n",
       "      <th>K_wreck</th>\n",
       "      <th>K_wreckage</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>16</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>18</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>19</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>23</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>24</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>25</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>26</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>28</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>31</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>32</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>33</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>34</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>36</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>37</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>38</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>39</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>40</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>41</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7583</th>\n",
       "      <td>10835</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7584</th>\n",
       "      <td>10837</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7585</th>\n",
       "      <td>10839</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7586</th>\n",
       "      <td>10840</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7587</th>\n",
       "      <td>10841</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7588</th>\n",
       "      <td>10842</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7589</th>\n",
       "      <td>10843</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7590</th>\n",
       "      <td>10844</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7591</th>\n",
       "      <td>10846</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7592</th>\n",
       "      <td>10847</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7593</th>\n",
       "      <td>10848</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7594</th>\n",
       "      <td>10849</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7595</th>\n",
       "      <td>10850</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7596</th>\n",
       "      <td>10851</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7597</th>\n",
       "      <td>10852</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7598</th>\n",
       "      <td>10853</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7599</th>\n",
       "      <td>10854</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7600</th>\n",
       "      <td>10855</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7601</th>\n",
       "      <td>10859</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7602</th>\n",
       "      <td>10860</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7603</th>\n",
       "      <td>10862</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7604</th>\n",
       "      <td>10863</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7605</th>\n",
       "      <td>10864</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7606</th>\n",
       "      <td>10866</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7607</th>\n",
       "      <td>10867</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7608</th>\n",
       "      <td>10869</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7609</th>\n",
       "      <td>10870</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7610</th>\n",
       "      <td>10871</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7611</th>\n",
       "      <td>10872</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7612</th>\n",
       "      <td>10873</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7613 rows × 186 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         id  target  K_ablaze  K_accident  K_aftershock  K_airplane accident  \\\n",
       "0         1       1         0           0             0                    0   \n",
       "1         4       1         0           0             0                    0   \n",
       "2         5       1         0           0             0                    0   \n",
       "3         6       1         0           0             0                    0   \n",
       "4         7       1         0           0             0                    0   \n",
       "5         8       1         0           0             0                    0   \n",
       "6        10       1         0           0             0                    0   \n",
       "7        13       1         0           0             0                    0   \n",
       "8        14       1         0           0             0                    0   \n",
       "9        15       1         0           0             0                    0   \n",
       "10       16       1         0           0             0                    0   \n",
       "11       17       1         0           0             0                    0   \n",
       "12       18       1         0           0             0                    0   \n",
       "13       19       1         0           0             0                    0   \n",
       "14       20       1         0           0             0                    0   \n",
       "15       23       0         0           0             0                    0   \n",
       "16       24       0         0           0             0                    0   \n",
       "17       25       0         0           0             0                    0   \n",
       "18       26       0         0           0             0                    0   \n",
       "19       28       0         0           0             0                    0   \n",
       "20       31       0         0           0             0                    0   \n",
       "21       32       0         0           0             0                    0   \n",
       "22       33       0         0           0             0                    0   \n",
       "23       34       0         0           0             0                    0   \n",
       "24       36       0         0           0             0                    0   \n",
       "25       37       0         0           0             0                    0   \n",
       "26       38       0         0           0             0                    0   \n",
       "27       39       0         0           0             0                    0   \n",
       "28       40       0         0           0             0                    0   \n",
       "29       41       0         0           0             0                    0   \n",
       "...     ...     ...       ...         ...           ...                  ...   \n",
       "7583  10835       1         0           0             0                    0   \n",
       "7584  10837       0         0           0             0                    0   \n",
       "7585  10839       1         0           0             0                    0   \n",
       "7586  10840       1         0           0             0                    0   \n",
       "7587  10841       0         0           0             0                    0   \n",
       "7588  10842       1         0           0             0                    0   \n",
       "7589  10843       1         0           0             0                    0   \n",
       "7590  10844       1         0           0             0                    0   \n",
       "7591  10846       1         0           0             0                    0   \n",
       "7592  10847       1         0           0             0                    0   \n",
       "7593  10848       0         0           0             0                    0   \n",
       "7594  10849       1         0           0             0                    0   \n",
       "7595  10850       1         0           0             0                    0   \n",
       "7596  10851       1         0           0             0                    0   \n",
       "7597  10852       1         0           0             0                    0   \n",
       "7598  10853       1         0           0             0                    0   \n",
       "7599  10854       1         0           0             0                    0   \n",
       "7600  10855       1         0           0             0                    0   \n",
       "7601  10859       1         0           0             0                    0   \n",
       "7602  10860       1         0           0             0                    0   \n",
       "7603  10862       1         0           0             0                    0   \n",
       "7604  10863       1         0           0             0                    0   \n",
       "7605  10864       1         0           0             0                    0   \n",
       "7606  10866       1         0           0             0                    0   \n",
       "7607  10867       1         0           0             0                    0   \n",
       "7608  10869       1         0           0             0                    0   \n",
       "7609  10870       1         0           0             0                    0   \n",
       "7610  10871       1         0           0             0                    0   \n",
       "7611  10872       1         0           0             0                    0   \n",
       "7612  10873       1         0           0             0                    0   \n",
       "\n",
       "      K_ambulance  K_annihilate  K_annihilation  K_apocalypse  ...  K_volcano  \\\n",
       "0               0             0               0             0  ...          0   \n",
       "1               0             0               0             0  ...          0   \n",
       "2               0             0               0             0  ...          0   \n",
       "3               0             0               0             0  ...          0   \n",
       "4               0             0               0             0  ...          0   \n",
       "5               0             0               0             0  ...          0   \n",
       "6               0             0               0             0  ...          0   \n",
       "7               0             0               0             0  ...          0   \n",
       "8               0             0               0             0  ...          0   \n",
       "9               0             0               0             0  ...          0   \n",
       "10              0             0               0             0  ...          0   \n",
       "11              0             0               0             0  ...          0   \n",
       "12              0             0               0             0  ...          0   \n",
       "13              0             0               0             0  ...          0   \n",
       "14              0             0               0             0  ...          0   \n",
       "15              0             0               0             0  ...          0   \n",
       "16              0             0               0             0  ...          0   \n",
       "17              0             0               0             0  ...          0   \n",
       "18              0             0               0             0  ...          0   \n",
       "19              0             0               0             0  ...          0   \n",
       "20              0             0               0             0  ...          0   \n",
       "21              0             0               0             0  ...          0   \n",
       "22              0             0               0             0  ...          0   \n",
       "23              0             0               0             0  ...          0   \n",
       "24              0             0               0             0  ...          0   \n",
       "25              0             0               0             0  ...          0   \n",
       "26              0             0               0             0  ...          0   \n",
       "27              0             0               0             0  ...          0   \n",
       "28              0             0               0             0  ...          0   \n",
       "29              0             0               0             0  ...          0   \n",
       "...           ...           ...             ...           ...  ...        ...   \n",
       "7583            0             0               0             0  ...          0   \n",
       "7584            0             0               0             0  ...          0   \n",
       "7585            0             0               0             0  ...          0   \n",
       "7586            0             0               0             0  ...          0   \n",
       "7587            0             0               0             0  ...          0   \n",
       "7588            0             0               0             0  ...          0   \n",
       "7589            0             0               0             0  ...          0   \n",
       "7590            0             0               0             0  ...          0   \n",
       "7591            0             0               0             0  ...          0   \n",
       "7592            0             0               0             0  ...          0   \n",
       "7593            0             0               0             0  ...          0   \n",
       "7594            0             0               0             0  ...          0   \n",
       "7595            0             0               0             0  ...          0   \n",
       "7596            0             0               0             0  ...          0   \n",
       "7597            0             0               0             0  ...          0   \n",
       "7598            0             0               0             0  ...          0   \n",
       "7599            0             0               0             0  ...          0   \n",
       "7600            0             0               0             0  ...          0   \n",
       "7601            0             0               0             0  ...          0   \n",
       "7602            0             0               0             0  ...          0   \n",
       "7603            0             0               0             0  ...          0   \n",
       "7604            0             0               0             0  ...          0   \n",
       "7605            0             0               0             0  ...          0   \n",
       "7606            0             0               0             0  ...          0   \n",
       "7607            0             0               0             0  ...          0   \n",
       "7608            0             0               0             0  ...          0   \n",
       "7609            0             0               0             0  ...          0   \n",
       "7610            0             0               0             0  ...          1   \n",
       "7611            0             0               0             0  ...          0   \n",
       "7612            0             0               0             0  ...          0   \n",
       "\n",
       "      K_war zone  K_weapon  K_whirlwind  K_wild fire  K_wildfire  K_windstorm  \\\n",
       "0              0         0            0            0           0            0   \n",
       "1              0         0            0            0           0            0   \n",
       "2              0         0            0            0           0            0   \n",
       "3              0         0            0            0           1            0   \n",
       "4              0         0            0            0           1            0   \n",
       "5              0         0            0            0           1            0   \n",
       "6              0         0            0            0           0            0   \n",
       "7              0         0            0            0           0            0   \n",
       "8              0         0            0            0           0            0   \n",
       "9              0         0            0            0           0            0   \n",
       "10             0         0            0            0           0            0   \n",
       "11             0         0            0            0           0            0   \n",
       "12             0         0            0            0           0            0   \n",
       "13             0         0            0            0           0            0   \n",
       "14             0         0            0            0           0            0   \n",
       "15             0         0            0            0           0            0   \n",
       "16             0         0            0            0           0            0   \n",
       "17             0         0            0            0           0            0   \n",
       "18             0         0            0            0           0            0   \n",
       "19             0         0            0            0           0            0   \n",
       "20             0         0            0            0           0            0   \n",
       "21             0         0            0            0           0            0   \n",
       "22             0         0            0            0           0            0   \n",
       "23             0         0            0            0           0            0   \n",
       "24             0         0            0            0           0            0   \n",
       "25             0         0            0            0           0            0   \n",
       "26             0         0            0            0           0            0   \n",
       "27             0         0            0            0           0            0   \n",
       "28             0         0            0            0           0            0   \n",
       "29             0         0            0            0           0            0   \n",
       "...          ...       ...          ...          ...         ...          ...   \n",
       "7583           0         0            0            0           0            0   \n",
       "7584           0         0            0            0           0            0   \n",
       "7585           0         0            0            0           0            0   \n",
       "7586           0         0            0            0           0            0   \n",
       "7587           0         0            0            0           0            0   \n",
       "7588           0         0            0            0           0            0   \n",
       "7589           0         0            0            0           0            0   \n",
       "7590           0         0            0            0           0            0   \n",
       "7591           0         0            0            0           0            0   \n",
       "7592           0         0            0            0           0            0   \n",
       "7593           0         0            0            0           0            0   \n",
       "7594           0         0            0            0           0            0   \n",
       "7595           0         0            0            0           0            0   \n",
       "7596           0         0            0            0           0            0   \n",
       "7597           0         0            0            0           0            0   \n",
       "7598           0         0            0            0           0            0   \n",
       "7599           0         0            0            0           0            0   \n",
       "7600           0         0            0            0           0            0   \n",
       "7601           0         0            0            0           0            0   \n",
       "7602           0         0            0            0           0            0   \n",
       "7603           0         0            0            0           0            0   \n",
       "7604           0         0            0            0           0            0   \n",
       "7605           0         0            0            0           0            0   \n",
       "7606           0         0            0            0           0            0   \n",
       "7607           0         0            0            0           0            0   \n",
       "7608           0         0            0            0           0            0   \n",
       "7609           0         0            0            1           0            0   \n",
       "7610           0         0            0            0           0            0   \n",
       "7611           0         0            0            0           0            0   \n",
       "7612           0         0            0            0           1            0   \n",
       "\n",
       "      K_wound  K_wreck  K_wreckage  \n",
       "0           0        0           0  \n",
       "1           0        0           0  \n",
       "2           0        0           0  \n",
       "3           0        0           0  \n",
       "4           0        0           0  \n",
       "5           0        0           0  \n",
       "6           0        0           0  \n",
       "7           0        0           0  \n",
       "8           0        0           0  \n",
       "9           0        0           0  \n",
       "10          0        0           0  \n",
       "11          0        0           0  \n",
       "12          0        0           0  \n",
       "13          0        0           0  \n",
       "14          0        0           0  \n",
       "15          0        0           0  \n",
       "16          0        0           0  \n",
       "17          0        0           0  \n",
       "18          0        0           0  \n",
       "19          0        0           0  \n",
       "20          0        0           0  \n",
       "21          0        0           0  \n",
       "22          0        0           0  \n",
       "23          0        0           0  \n",
       "24          0        0           0  \n",
       "25          0        0           0  \n",
       "26          0        0           0  \n",
       "27          0        0           0  \n",
       "28          0        0           0  \n",
       "29          0        0           0  \n",
       "...       ...      ...         ...  \n",
       "7583        0        0           0  \n",
       "7584        0        0           0  \n",
       "7585        0        0           0  \n",
       "7586        0        0           0  \n",
       "7587        0        0           0  \n",
       "7588        0        0           0  \n",
       "7589        0        0           0  \n",
       "7590        0        0           0  \n",
       "7591        0        0           0  \n",
       "7592        0        0           0  \n",
       "7593        0        0           0  \n",
       "7594        0        0           0  \n",
       "7595        0        0           0  \n",
       "7596        0        0           0  \n",
       "7597        0        0           0  \n",
       "7598        0        0           0  \n",
       "7599        0        0           0  \n",
       "7600        0        0           0  \n",
       "7601        0        0           0  \n",
       "7602        0        0           0  \n",
       "7603        0        0           0  \n",
       "7604        0        0           0  \n",
       "7605        0        0           0  \n",
       "7606        0        0           0  \n",
       "7607        0        0           0  \n",
       "7608        0        0           0  \n",
       "7609        0        0           0  \n",
       "7610        0        0           0  \n",
       "7611        0        0           0  \n",
       "7612        0        0           0  \n",
       "\n",
       "[7613 rows x 186 columns]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mlb = MultiLabelBinarizer()\n",
    "\n",
    "keywords_categorical_features = keywords_categorical_features.join(pd.DataFrame(mlb.fit_transform(keywords_categorical_features.pop('keywords_in_text')),\n",
    "                          columns=mlb.classes_,\n",
    "                          index=keywords_categorical_features.index).add_prefix('K_'))\n",
    "keywords_categorical_features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Agrego features a set de test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>keywords_in_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>[crash]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>[earthquake]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>[fire, forest fire]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9</td>\n",
       "      <td>[apocalypse, fire, wildfire]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>11</td>\n",
       "      <td>[typhoon]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id              keywords_in_text\n",
       "0   0                       [crash]\n",
       "1   2                  [earthquake]\n",
       "2   3           [fire, forest fire]\n",
       "3   9  [apocalypse, fire, wildfire]\n",
       "4  11                     [typhoon]"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_numerical_keywords   = test_keywords.copy()\n",
    "test_categorical_keywords = test_keywords.copy()\n",
    "test_numerical_keywords   = keywords_agregar_features_numericos(test_numerical_keywords,False).drop(columns=['location','text','keyword'])\n",
    "test_categorical_keywords = keywords_agregar_features_categoricos(test_categorical_keywords,False).drop(columns=['location','text','keyword'])\n",
    "test_categorical_keywords.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/anichu/Programs/anaconda3/lib/python3.6/site-packages/sklearn/preprocessing/_label.py:987: UserWarning: unknown class(es) ['cliff fall'] will be ignored\n",
      "  .format(sorted(unknown, key=str)))\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>K_ablaze</th>\n",
       "      <th>K_accident</th>\n",
       "      <th>K_aftershock</th>\n",
       "      <th>K_airplane accident</th>\n",
       "      <th>K_ambulance</th>\n",
       "      <th>K_annihilate</th>\n",
       "      <th>K_annihilation</th>\n",
       "      <th>K_apocalypse</th>\n",
       "      <th>K_armageddon</th>\n",
       "      <th>...</th>\n",
       "      <th>K_volcano</th>\n",
       "      <th>K_war zone</th>\n",
       "      <th>K_weapon</th>\n",
       "      <th>K_whirlwind</th>\n",
       "      <th>K_wild fire</th>\n",
       "      <th>K_wildfire</th>\n",
       "      <th>K_windstorm</th>\n",
       "      <th>K_wound</th>\n",
       "      <th>K_wreck</th>\n",
       "      <th>K_wreckage</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 185 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  K_ablaze  K_accident  K_aftershock  K_airplane accident  K_ambulance  \\\n",
       "0   0         0           0             0                    0            0   \n",
       "1   2         0           0             0                    0            0   \n",
       "2   3         0           0             0                    0            0   \n",
       "3   9         0           0             0                    0            0   \n",
       "4  11         0           0             0                    0            0   \n",
       "\n",
       "   K_annihilate  K_annihilation  K_apocalypse  K_armageddon  ...  K_volcano  \\\n",
       "0             0               0             0             0  ...          0   \n",
       "1             0               0             0             0  ...          0   \n",
       "2             0               0             0             0  ...          0   \n",
       "3             0               0             1             0  ...          0   \n",
       "4             0               0             0             0  ...          0   \n",
       "\n",
       "   K_war zone  K_weapon  K_whirlwind  K_wild fire  K_wildfire  K_windstorm  \\\n",
       "0           0         0            0            0           0            0   \n",
       "1           0         0            0            0           0            0   \n",
       "2           0         0            0            0           0            0   \n",
       "3           0         0            0            0           1            0   \n",
       "4           0         0            0            0           0            0   \n",
       "\n",
       "   K_wound  K_wreck  K_wreckage  \n",
       "0        0        0           0  \n",
       "1        0        0           0  \n",
       "2        0        0           0  \n",
       "3        0        0           0  \n",
       "4        0        0           0  \n",
       "\n",
       "[5 rows x 185 columns]"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_categorical_keywords = test_categorical_keywords.join(pd.DataFrame(mlb.transform(test_categorical_keywords.pop('keywords_in_text')),\n",
    "                          columns=mlb.classes_,\n",
    "                          index=test_categorical_keywords.index).add_prefix('K_'))\n",
    "test_categorical_keywords.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Dataframes finales"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "#keywords_numerical_features.to_csv('features/train/features_keywords_numerical.csv', encoding='utf-8')\n",
    "#keywords_categorical_features.to_csv('features/train/features_keywords_in_text_encoded.csv', encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "#test_numerical_keywords.to_csv('features/test/features_keywords_numerical.csv', encoding='utf-8')\n",
    "#test_categorical_keywords.to_csv('features/test/features_keywords_in_text_encoded.csv', encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "#corrmap = df_keywords.corr()\n",
    "#fig = plt.figure(figsize=(20,10))\n",
    "\n",
    "#sns.heatmap(corrmap, vmax=.8, square=True)\n",
    "#plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.2 Text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 261,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Funciones auxiliares\n",
    "stopword = stopwords.words('english')\n",
    "\n",
    "def stop(text):\n",
    "    return [w for w in text.split() if w in stopword]\n",
    "\n",
    "def length(text):\n",
    "    return(np.mean([len(w) for w in text.split()]))\n",
    "\n",
    "def punct(text):\n",
    "    return [c for c in text if c in string.punctuation]\n",
    "\n",
    "def title(text):\n",
    "    return [w for w in text.split() if w.istitle()]\n",
    "\n",
    "def upper_list(text):\n",
    "    return [w for w in text.split() if w.isupper()]\n",
    "\n",
    "def lower_list(text):\n",
    "    return [w for w in text.split() if w.islower()]\n",
    "\n",
    "def count_special_char(text):\n",
    "    special_char = 0\n",
    "    text = text.replace(\" \", \"\")\n",
    "    for i in range(0, len(text)):\n",
    "        ch = text[i]\n",
    "        if (ch.isalpha()):  \n",
    "            continue\n",
    "        elif (ch.isdigit()):\n",
    "            continue\n",
    "        else:\n",
    "            special_char += 1\n",
    "    return special_char\n",
    "\n",
    "def syllables(text):\n",
    "    count = 0\n",
    "    vowels = 'aeiouy'\n",
    "    for word in text:\n",
    "        word= word.lower()\n",
    "        if word[0] in vowels:\n",
    "            count +=1\n",
    "        for index in range(1,len(word)):\n",
    "            if word[index] in vowels and word[index-1] not in vowels:\n",
    "                count +=1\n",
    "        if word.endswith('e'):\n",
    "            count -= 1\n",
    "        if word.endswith('le'):\n",
    "            count+=1\n",
    "        if count == 0:\n",
    "            count +=1\n",
    "    return count\n",
    "\n",
    "def quitar_link_twitter(tweet):\n",
    "    res = []\n",
    "    for w in tweet.split(\" \"):\n",
    "        if ((\"http\" not in w) and (\"https\" not in w)):\n",
    "            res.append(w)\n",
    "    return (\" \").join(res)\n",
    "\n",
    "def quitar_mencion_twitter(tweet):\n",
    "    res = []\n",
    "    for w in tweet.split(\" \"):\n",
    "        if(\"@\" not in w):\n",
    "            res.append(w)\n",
    "    return (\" \").join(res)\n",
    "\n",
    "def getHashtags(tweet):\n",
    "    pat = re.compile(r\"#(\\w+)\")\n",
    "    return pat.findall(tweet)\n",
    "\n",
    "def getTaggedUsers(tweet):\n",
    "    pat = re.compile(r\"@(\\w+)\")\n",
    "    return pat.findall(tweet)\n",
    "\n",
    "def agregar_features_cantidad_palabras(df_og):\n",
    "    df_og['text'] = df_og['text'].apply(lambda x: quitar_link_twitter(x))\n",
    "    df_og['text'] = df_og['text'].apply(lambda x: quitar_mencion_twitter(x))\n",
    "    palabras_list = [df_og]\n",
    "    for df in palabras_list:\n",
    "        # Número de palabras usadas\n",
    "        df['#palabras'] = df['text'].apply(lambda x: len(x.split()))\n",
    "        # Número de palabras únicas\n",
    "        df['#palabras_unicas'] =  df['text'].apply(lambda x: len(set(x.split())))\n",
    "        # Número de caracteres\n",
    "        df['#caracteres'] =  df['text'].apply(lambda x: len(x))\n",
    "        # Número de stopwords\n",
    "        df['#stopwords'] = df['text'].apply(lambda x: len(stop(x)))\n",
    "        # Número de caracteres de puntuación\n",
    "        df['#puntuacion'] = df['text'].apply(lambda x: len(punct(x)))\n",
    "        # Número de palabras Capitalizadas\n",
    "        df['#capitalize'] = df['text'].apply(lambda x: len(title(x)))\n",
    "        # Número de palabras MAYUSCULAS\n",
    "        df['#mayusculas'] = df['text'].apply(lambda x: len(upper_list(x)))\n",
    "        # Número de silabas\n",
    "        df['#silabas'] = df['text'].apply(lambda x: syllables(x))\n",
    "        # Promedio de longitud del tweet\n",
    "        df['promedio_len_word'] = df['text'].apply(lambda x: length(x))\n",
    "        # Número de caracteres especiales\n",
    "        df['#caracteres_especiales'] = df['text'].apply(lambda x: count_special_char(x))\n",
    "    return df\n",
    "\n",
    "def getUniqueHashtagsList(hashtags):\n",
    "    hashtagList = []\n",
    "    for hashtag in hashtags:\n",
    "        for h in hashtag:\n",
    "            if h not in hashtagList:\n",
    "                hashtagList.append(h.lower())\n",
    "    return hashtagList\n",
    "\n",
    "def getHashtagsList(hashtags):\n",
    "    hashtagList = []\n",
    "    for hashtag in hashtags:\n",
    "        for h in hashtag:\n",
    "            hashtagList.append(h.lower())\n",
    "    return hashtagList\n",
    "\n",
    "def getMostFrequentHashtag(hashtags):\n",
    "    maxValue = 0\n",
    "    finalHashtag = ''\n",
    "    if (len(hashtags) > 0):\n",
    "        for hashtag in hashtags:\n",
    "            hashtag = hashtag.lower()\n",
    "            if (hashtagDict.get(hashtag) > maxValue):\n",
    "                finalHashtag = hashtag\n",
    "                maxValue = hashtagDict.get(hashtag)\n",
    "\n",
    "    return finalHashtag\n",
    "\n",
    "def getHashtagFrequency(hashtag):\n",
    "    if hashtag != '':\n",
    "        return hashtagDict.get(hashtag)/len(hashtagDict)\n",
    "    else:\n",
    "        return 0\n",
    "    \n",
    "def getUniqueUsersList(users):\n",
    "    userList = []\n",
    "    for user in users:\n",
    "        for u in user:\n",
    "            if u not in userList:\n",
    "                userList.append(u.lower())\n",
    "    return userList\n",
    "\n",
    "def getUsersList(users):\n",
    "    userList = []\n",
    "    for user in users:\n",
    "        for u in user:\n",
    "            userList.append(u.lower())\n",
    "    return userList\n",
    "\n",
    "def hasRelevantTaggedUser(users):\n",
    "    minValue = 5\n",
    "    #finalUser = ''\n",
    "    if (len(users) > 0):\n",
    "        for user in users:\n",
    "            user = user.lower()\n",
    "            if (userDict.get(user) > minValue):\n",
    "                return True\n",
    "    return False\n",
    "\n",
    "def text_agregar_features_numericos(df):\n",
    "    df = agregar_features_cantidad_palabras(df.copy())\n",
    "    df['#palabras_binned'] = pd.qcut(df['#palabras'], q=4, labels=False)\n",
    "    df['#palabras_unicas_binned'] = pd.qcut(df['#palabras_unicas'], q=4, labels=False)\n",
    "    df['#caracteres_binned'] = pd.qcut(df['#caracteres'], q=5, labels=False)\n",
    "    df['#stopwords_binned']  = pd.qcut(df['#stopwords'], q=3, labels=False)\n",
    "    df['#puntuacion_binned'] = pd.qcut(df['#puntuacion'], q=3, labels=False)\n",
    "    df['#capitalize_binned'] = pd.qcut(df['#capitalize'], q=3, labels=False)\n",
    "    df['#mayusculas_binned'] = pd.qcut(df['#mayusculas'], q=2, labels=False, duplicates='drop')\n",
    "    df['#silabas_binned']    = pd.qcut(df['#silabas'], q=4, labels=False)\n",
    "    df['#caracteres_especiales_binned'] = pd.qcut(df['#caracteres_especiales'], q=3, labels=False)\n",
    "    return df\n",
    "\n",
    "def hashtags_agregar_features_numericos(df):\n",
    "    df['hashtags']    = df.text.apply(lambda x: getHashtags(x))\n",
    "    df['#hashtags']   = df.hashtags.apply(lambda x: len(x))\n",
    "    df['has_hashtag'] = (df['#hashtags']>0).astype(int)\n",
    "#    hashtags    = getUniqueHashtagsList(df.hashtags)\n",
    "#    hashtagDict = Counter(getHashtagsList(df.hashtags))\n",
    "#    df['relevant_hashtag']      = df.hashtags.apply(lambda x: getMostFrequentHashtag(x))\n",
    "#    df['freq_relevant_hashtag'] = df.relevant_hashtag.apply(lambda x:getHashtagFrequency(x))\n",
    "    df = df.drop(columns=['hashtags'])\n",
    "    return df\n",
    "\n",
    "def hashtags_agregar_features_categoricos(df):\n",
    "    df['hashtags'] = df.text.apply(lambda x: getHashtags(x))\n",
    "    #df_text['relevant_hashtag'] = df_text.hashtags.apply(lambda x: getMostFrequentHashtag(x))\n",
    "    return df\n",
    "\n",
    "def tagged_users_agregar_features_numericos(df):\n",
    "    df['tagged_users'] = df.text.apply(lambda x: getTaggedUsers(x))\n",
    "    df['#tagged_users'] = df.tagged_users.apply(lambda x: len(x))\n",
    "#    users = getUniqueUsersList(df.tagged_users)\n",
    "#    userDict = Counter(getUsersList(df.tagged_users))\n",
    "#    df['has_tagged_user'] = (df['#tagged_users']>0).astype(int)\n",
    "#    df['has_relevant_tagged_user'] = (df.tagged_users.apply(lambda x:hasRelevantTaggedUser(x))).astype(int)\n",
    "    df = df.drop(columns=['tagged_users'])\n",
    "    return df\n",
    "\n",
    "def tagged_users_agregar_features_categoricos(df):\n",
    "    df['tagged_users'] = df.text.apply(lambda x: getTaggedUsers(x))\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_text = train.copy()\n",
    "df_text = df_text.drop(columns=['keyword','location'])\n",
    "df_text.text = df_text.text.apply(clean_text_non_strict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test_text = test.copy()\n",
    "df_test_text = df_test_text.drop(columns=['keyword','location'])\n",
    "df_test_text.text = df_test_text.text.apply(clean_text_non_strict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Features NUMERICAL TEXT"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Features TEXT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 264,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>target</th>\n",
       "      <th>#palabras</th>\n",
       "      <th>#palabras_unicas</th>\n",
       "      <th>#caracteres</th>\n",
       "      <th>#stopwords</th>\n",
       "      <th>#puntuacion</th>\n",
       "      <th>#capitalize</th>\n",
       "      <th>#mayusculas</th>\n",
       "      <th>#silabas</th>\n",
       "      <th>...</th>\n",
       "      <th>#caracteres_especiales</th>\n",
       "      <th>#palabras_binned</th>\n",
       "      <th>#palabras_unicas_binned</th>\n",
       "      <th>#caracteres_binned</th>\n",
       "      <th>#stopwords_binned</th>\n",
       "      <th>#puntuacion_binned</th>\n",
       "      <th>#capitalize_binned</th>\n",
       "      <th>#mayusculas_binned</th>\n",
       "      <th>#silabas_binned</th>\n",
       "      <th>#caracteres_especiales_binned</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>13</td>\n",
       "      <td>71</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>19</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>40</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>22</td>\n",
       "      <td>20</td>\n",
       "      <td>135</td>\n",
       "      <td>9</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>27</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>67</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>17</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>16</td>\n",
       "      <td>15</td>\n",
       "      <td>90</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>24</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  target  #palabras  #palabras_unicas  #caracteres  #stopwords  \\\n",
       "0   1       1         13                13           71           5   \n",
       "1   4       1          7                 7           40           0   \n",
       "2   5       1         22                20          135           9   \n",
       "3   6       1          8                 8           67           1   \n",
       "4   7       1         16                15           90           6   \n",
       "\n",
       "   #puntuacion  #capitalize  #mayusculas  #silabas  ...  \\\n",
       "0            1            5            1        19  ...   \n",
       "1            1            5            0        10  ...   \n",
       "2            3            2            0        27  ...   \n",
       "3            2            1            0        17  ...   \n",
       "4            2            3            0        24  ...   \n",
       "\n",
       "   #caracteres_especiales  #palabras_binned  #palabras_unicas_binned  \\\n",
       "0                       1                 1                        1   \n",
       "1                       1                 0                        0   \n",
       "2                       3                 3                        3   \n",
       "3                       2                 0                        0   \n",
       "4                       2                 2                        2   \n",
       "\n",
       "   #caracteres_binned  #stopwords_binned  #puntuacion_binned  \\\n",
       "0                   1                  1                   0   \n",
       "1                   0                  0                   0   \n",
       "2                   4                  2                   1   \n",
       "3                   1                  0                   0   \n",
       "4                   2                  2                   0   \n",
       "\n",
       "   #capitalize_binned  #mayusculas_binned  #silabas_binned  \\\n",
       "0                   2                   0                1   \n",
       "1                   2                   0                0   \n",
       "2                   1                   0                3   \n",
       "3                   0                   0                1   \n",
       "4                   1                   0                2   \n",
       "\n",
       "   #caracteres_especiales_binned  \n",
       "0                              0  \n",
       "1                              0  \n",
       "2                              1  \n",
       "3                              0  \n",
       "4                              0  \n",
       "\n",
       "[5 rows x 21 columns]"
      ]
     },
     "execution_count": 264,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text_numerical_features = df_text.copy()\n",
    "text_numerical_features = text_agregar_features_numericos(text_numerical_features).drop(columns=['text'])\n",
    "text_numerical_features.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_text_numerical_features = df_test_text.copy()\n",
    "test_text_numerical_features = text_agregar_features_numericos(test_text_numerical_features).drop(columns=['text'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Features ARROBAS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "tagged_users_numerical_features = df_text.copy()\n",
    "tagged_users_numerical_features = tagged_users_agregar_features_numericos(tagged_users_numerical_features).drop(columns=['text'])\n",
    "tagged_users_categorical_features = df_text.copy()\n",
    "tagged_users_categorical_features = tagged_users_agregar_features_categoricos(tagged_users_categorical_features).drop(columns=['text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>#tagged_users</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>21</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>22</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>27</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>29</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>30</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>35</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>42</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>43</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>45</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>46</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>47</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>51</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>58</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>60</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>69</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>70</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>72</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>75</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>84</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>87</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>88</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>90</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>94</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>99</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3233</th>\n",
       "      <td>10756</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3234</th>\n",
       "      <td>10757</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3235</th>\n",
       "      <td>10758</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3236</th>\n",
       "      <td>10761</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3237</th>\n",
       "      <td>10762</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3238</th>\n",
       "      <td>10773</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3239</th>\n",
       "      <td>10778</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3240</th>\n",
       "      <td>10781</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3241</th>\n",
       "      <td>10791</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3242</th>\n",
       "      <td>10792</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3243</th>\n",
       "      <td>10796</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3244</th>\n",
       "      <td>10797</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3245</th>\n",
       "      <td>10801</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3246</th>\n",
       "      <td>10804</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3247</th>\n",
       "      <td>10806</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3248</th>\n",
       "      <td>10807</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3249</th>\n",
       "      <td>10816</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3250</th>\n",
       "      <td>10820</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3251</th>\n",
       "      <td>10828</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3252</th>\n",
       "      <td>10836</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3253</th>\n",
       "      <td>10838</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3254</th>\n",
       "      <td>10845</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3255</th>\n",
       "      <td>10856</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3256</th>\n",
       "      <td>10857</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3257</th>\n",
       "      <td>10858</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3258</th>\n",
       "      <td>10861</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3259</th>\n",
       "      <td>10865</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3260</th>\n",
       "      <td>10868</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3261</th>\n",
       "      <td>10874</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3262</th>\n",
       "      <td>10875</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3263 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         id  #tagged_users\n",
       "0         0              0\n",
       "1         2              0\n",
       "2         3              0\n",
       "3         9              0\n",
       "4        11              0\n",
       "5        12              0\n",
       "6        21              0\n",
       "7        22              0\n",
       "8        27              0\n",
       "9        29              0\n",
       "10       30              0\n",
       "11       35              0\n",
       "12       42              0\n",
       "13       43              0\n",
       "14       45              0\n",
       "15       46              0\n",
       "16       47              1\n",
       "17       51              0\n",
       "18       58              0\n",
       "19       60              2\n",
       "20       69              0\n",
       "21       70              1\n",
       "22       72              1\n",
       "23       75              0\n",
       "24       84              0\n",
       "25       87              1\n",
       "26       88              0\n",
       "27       90              1\n",
       "28       94              2\n",
       "29       99              0\n",
       "...     ...            ...\n",
       "3233  10756              0\n",
       "3234  10757              0\n",
       "3235  10758              0\n",
       "3236  10761              0\n",
       "3237  10762              1\n",
       "3238  10773              1\n",
       "3239  10778              0\n",
       "3240  10781              1\n",
       "3241  10791              0\n",
       "3242  10792              0\n",
       "3243  10796              0\n",
       "3244  10797              1\n",
       "3245  10801              0\n",
       "3246  10804              1\n",
       "3247  10806              0\n",
       "3248  10807              0\n",
       "3249  10816              1\n",
       "3250  10820              1\n",
       "3251  10828              0\n",
       "3252  10836              0\n",
       "3253  10838              0\n",
       "3254  10845              0\n",
       "3255  10856              1\n",
       "3256  10857              0\n",
       "3257  10858              0\n",
       "3258  10861              0\n",
       "3259  10865              0\n",
       "3260  10868              0\n",
       "3261  10874              0\n",
       "3262  10875              0\n",
       "\n",
       "[3263 rows x 2 columns]"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_tagged_users_numerical_features = df_test_text.copy()\n",
    "test_tagged_users_numerical_features = tagged_users_agregar_features_numericos(test_tagged_users_numerical_features).drop(columns=['text'])\n",
    "test_tagged_users_categorical_features = df_test_text.copy()\n",
    "test_tagged_users_categorical_features = tagged_users_agregar_features_categoricos(test_tagged_users_categorical_features).drop(columns=['text'])\n",
    "test_tagged_users_numerical_features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "TF-IDF - ARROBAS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1061thetwister</th>\n",
       "      <th>115film</th>\n",
       "      <th>15moferadio</th>\n",
       "      <th>1acd4900c1424d1</th>\n",
       "      <th>1rockstar62</th>\n",
       "      <th>20skyhawkmm20</th>\n",
       "      <th>233liveonline</th>\n",
       "      <th>2for1lapdances</th>\n",
       "      <th>4tiles</th>\n",
       "      <th>4tp_kotc</th>\n",
       "      <th>...</th>\n",
       "      <th>zaynmalik</th>\n",
       "      <th>zenande_mcfen</th>\n",
       "      <th>zeno001</th>\n",
       "      <th>zepp1978</th>\n",
       "      <th>zhenghxn</th>\n",
       "      <th>zojadelin</th>\n",
       "      <th>zomatoaus</th>\n",
       "      <th>zourryart</th>\n",
       "      <th>zxathetis</th>\n",
       "      <th>id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 2311 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   1061thetwister  115film  15moferadio  1acd4900c1424d1  1rockstar62  \\\n",
       "0             0.0      0.0          0.0              0.0          0.0   \n",
       "1             0.0      0.0          0.0              0.0          0.0   \n",
       "2             0.0      0.0          0.0              0.0          0.0   \n",
       "3             0.0      0.0          0.0              0.0          0.0   \n",
       "4             0.0      0.0          0.0              0.0          0.0   \n",
       "\n",
       "   20skyhawkmm20  233liveonline  2for1lapdances  4tiles  4tp_kotc  ...  \\\n",
       "0            0.0            0.0             0.0     0.0       0.0  ...   \n",
       "1            0.0            0.0             0.0     0.0       0.0  ...   \n",
       "2            0.0            0.0             0.0     0.0       0.0  ...   \n",
       "3            0.0            0.0             0.0     0.0       0.0  ...   \n",
       "4            0.0            0.0             0.0     0.0       0.0  ...   \n",
       "\n",
       "   zaynmalik  zenande_mcfen  zeno001  zepp1978  zhenghxn  zojadelin  \\\n",
       "0        0.0            0.0      0.0       0.0       0.0        0.0   \n",
       "1        0.0            0.0      0.0       0.0       0.0        0.0   \n",
       "2        0.0            0.0      0.0       0.0       0.0        0.0   \n",
       "3        0.0            0.0      0.0       0.0       0.0        0.0   \n",
       "4        0.0            0.0      0.0       0.0       0.0        0.0   \n",
       "\n",
       "   zomatoaus  zourryart  zxathetis  id  \n",
       "0        0.0        0.0        0.0   1  \n",
       "1        0.0        0.0        0.0   4  \n",
       "2        0.0        0.0        0.0   5  \n",
       "3        0.0        0.0        0.0   6  \n",
       "4        0.0        0.0        0.0   7  \n",
       "\n",
       "[5 rows x 2311 columns]"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tagged_users_categorical_features['tagged_users'] = tagged_users_categorical_features['tagged_users'].apply(lambda x:toText(x))\n",
    "tagged_index = tagged_users_categorical_features['tagged_users'].fillna('').index\n",
    "tagged_id = tagged_users_categorical_features['id'].fillna('')\n",
    "corpus = tagged_users_categorical_features['tagged_users'].fillna('')\n",
    "\n",
    "# Without Smoothing:\n",
    "tf_idf_vec = TfidfVectorizer(use_idf=True, \n",
    "                        smooth_idf=False,  \n",
    "                        ngram_range=(1,1),stop_words='english') # to use only  bigrams ngram_range=(2,2)\n",
    "#transform\n",
    "tf_idf_data = tf_idf_vec.fit_transform(corpus)\n",
    "\n",
    "#create dataframe\n",
    "df_tf_idf_tagged=pd.DataFrame(tf_idf_data.toarray(),columns=tf_idf_vec.get_feature_names())\n",
    "df_tf_idf_tagged.set_index(tagged_index)\n",
    "df_tf_idf_tagged['id'] = tagged_id\n",
    "\n",
    "df_tf_idf_tagged.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1061thetwister</th>\n",
       "      <th>115film</th>\n",
       "      <th>15moferadio</th>\n",
       "      <th>1acd4900c1424d1</th>\n",
       "      <th>1rockstar62</th>\n",
       "      <th>20skyhawkmm20</th>\n",
       "      <th>233liveonline</th>\n",
       "      <th>2for1lapdances</th>\n",
       "      <th>4tiles</th>\n",
       "      <th>4tp_kotc</th>\n",
       "      <th>...</th>\n",
       "      <th>zaynmalik</th>\n",
       "      <th>zenande_mcfen</th>\n",
       "      <th>zeno001</th>\n",
       "      <th>zepp1978</th>\n",
       "      <th>zhenghxn</th>\n",
       "      <th>zojadelin</th>\n",
       "      <th>zomatoaus</th>\n",
       "      <th>zourryart</th>\n",
       "      <th>zxathetis</th>\n",
       "      <th>id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 2311 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   1061thetwister  115film  15moferadio  1acd4900c1424d1  1rockstar62  \\\n",
       "0             0.0      0.0          0.0              0.0          0.0   \n",
       "1             0.0      0.0          0.0              0.0          0.0   \n",
       "2             0.0      0.0          0.0              0.0          0.0   \n",
       "3             0.0      0.0          0.0              0.0          0.0   \n",
       "4             0.0      0.0          0.0              0.0          0.0   \n",
       "\n",
       "   20skyhawkmm20  233liveonline  2for1lapdances  4tiles  4tp_kotc  ...  \\\n",
       "0            0.0            0.0             0.0     0.0       0.0  ...   \n",
       "1            0.0            0.0             0.0     0.0       0.0  ...   \n",
       "2            0.0            0.0             0.0     0.0       0.0  ...   \n",
       "3            0.0            0.0             0.0     0.0       0.0  ...   \n",
       "4            0.0            0.0             0.0     0.0       0.0  ...   \n",
       "\n",
       "   zaynmalik  zenande_mcfen  zeno001  zepp1978  zhenghxn  zojadelin  \\\n",
       "0        0.0            0.0      0.0       0.0       0.0        0.0   \n",
       "1        0.0            0.0      0.0       0.0       0.0        0.0   \n",
       "2        0.0            0.0      0.0       0.0       0.0        0.0   \n",
       "3        0.0            0.0      0.0       0.0       0.0        0.0   \n",
       "4        0.0            0.0      0.0       0.0       0.0        0.0   \n",
       "\n",
       "   zomatoaus  zourryart  zxathetis  id  \n",
       "0        0.0        0.0        0.0   0  \n",
       "1        0.0        0.0        0.0   2  \n",
       "2        0.0        0.0        0.0   3  \n",
       "3        0.0        0.0        0.0   9  \n",
       "4        0.0        0.0        0.0  11  \n",
       "\n",
       "[5 rows x 2311 columns]"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_tagged_users_categorical_features['tagged_users'] = test_tagged_users_categorical_features['tagged_users'].apply(lambda x:toText(x))\n",
    "test_tagged_index = test_tagged_users_categorical_features['tagged_users'].fillna('').index\n",
    "tagged_id = test_tagged_users_categorical_features['id'].fillna('')\n",
    "corpus    = test_tagged_users_categorical_features['tagged_users'].fillna('')\n",
    "\n",
    "#transform\n",
    "tf_idf_data = tf_idf_vec.transform(corpus)\n",
    "\n",
    "#create dataframe\n",
    "test_tf_idf_tagged=pd.DataFrame(tf_idf_data.toarray(),columns=tf_idf_vec.get_feature_names())\n",
    "test_tf_idf_tagged.set_index(test_tagged_index)\n",
    "test_tf_idf_tagged['id'] = tagged_id\n",
    "\n",
    "test_tf_idf_tagged.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1061thetwister</th>\n",
       "      <th>115film</th>\n",
       "      <th>15moferadio</th>\n",
       "      <th>1acd4900c1424d1</th>\n",
       "      <th>1rockstar62</th>\n",
       "      <th>20skyhawkmm20</th>\n",
       "      <th>233liveonline</th>\n",
       "      <th>2for1lapdances</th>\n",
       "      <th>4tiles</th>\n",
       "      <th>4tp_kotc</th>\n",
       "      <th>...</th>\n",
       "      <th>zaynmalik</th>\n",
       "      <th>zenande_mcfen</th>\n",
       "      <th>zeno001</th>\n",
       "      <th>zepp1978</th>\n",
       "      <th>zhenghxn</th>\n",
       "      <th>zojadelin</th>\n",
       "      <th>zomatoaus</th>\n",
       "      <th>zourryart</th>\n",
       "      <th>zxathetis</th>\n",
       "      <th>id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 2311 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   1061thetwister  115film  15moferadio  1acd4900c1424d1  1rockstar62  \\\n",
       "0             0.0      0.0          0.0              0.0          0.0   \n",
       "1             0.0      0.0          0.0              0.0          0.0   \n",
       "2             0.0      0.0          0.0              0.0          0.0   \n",
       "3             0.0      0.0          0.0              0.0          0.0   \n",
       "4             0.0      0.0          0.0              0.0          0.0   \n",
       "\n",
       "   20skyhawkmm20  233liveonline  2for1lapdances  4tiles  4tp_kotc  ...  \\\n",
       "0            0.0            0.0             0.0     0.0       0.0  ...   \n",
       "1            0.0            0.0             0.0     0.0       0.0  ...   \n",
       "2            0.0            0.0             0.0     0.0       0.0  ...   \n",
       "3            0.0            0.0             0.0     0.0       0.0  ...   \n",
       "4            0.0            0.0             0.0     0.0       0.0  ...   \n",
       "\n",
       "   zaynmalik  zenande_mcfen  zeno001  zepp1978  zhenghxn  zojadelin  \\\n",
       "0        0.0            0.0      0.0       0.0       0.0        0.0   \n",
       "1        0.0            0.0      0.0       0.0       0.0        0.0   \n",
       "2        0.0            0.0      0.0       0.0       0.0        0.0   \n",
       "3        0.0            0.0      0.0       0.0       0.0        0.0   \n",
       "4        0.0            0.0      0.0       0.0       0.0        0.0   \n",
       "\n",
       "   zomatoaus  zourryart  zxathetis  id  \n",
       "0        0.0        0.0        0.0   1  \n",
       "1        0.0        0.0        0.0   4  \n",
       "2        0.0        0.0        0.0   5  \n",
       "3        0.0        0.0        0.0   6  \n",
       "4        0.0        0.0        0.0   7  \n",
       "\n",
       "[5 rows x 2311 columns]"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tagged_index = tagged_users_categorical_features['tagged_users'].fillna('').index\n",
    "tagged_id = tagged_users_categorical_features['id'].fillna('')\n",
    "corpus = tagged_users_categorical_features['tagged_users'].fillna('')\n",
    "\n",
    "# With Smoothing:\n",
    "\n",
    "tf_idf_vec = TfidfVectorizer(use_idf=True, \n",
    "                        smooth_idf=True,  \n",
    "                        ngram_range=(1,1),stop_words='english') # to use only  bigrams ngram_range=(2,2)\n",
    "#transform\n",
    "tf_idf_data = tf_idf_vec.fit_transform(corpus)\n",
    "\n",
    "#create dataframe\n",
    "df_tf_idf_smooth_tagged=pd.DataFrame(tf_idf_data.toarray(),columns=tf_idf_vec.get_feature_names())\n",
    "df_tf_idf_smooth_tagged.set_index(tagged_index)\n",
    "df_tf_idf_smooth_tagged['id'] = tagged_id\n",
    "\n",
    "df_tf_idf_smooth_tagged.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1061thetwister</th>\n",
       "      <th>115film</th>\n",
       "      <th>15moferadio</th>\n",
       "      <th>1acd4900c1424d1</th>\n",
       "      <th>1rockstar62</th>\n",
       "      <th>20skyhawkmm20</th>\n",
       "      <th>233liveonline</th>\n",
       "      <th>2for1lapdances</th>\n",
       "      <th>4tiles</th>\n",
       "      <th>4tp_kotc</th>\n",
       "      <th>...</th>\n",
       "      <th>zaynmalik</th>\n",
       "      <th>zenande_mcfen</th>\n",
       "      <th>zeno001</th>\n",
       "      <th>zepp1978</th>\n",
       "      <th>zhenghxn</th>\n",
       "      <th>zojadelin</th>\n",
       "      <th>zomatoaus</th>\n",
       "      <th>zourryart</th>\n",
       "      <th>zxathetis</th>\n",
       "      <th>id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 2311 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   1061thetwister  115film  15moferadio  1acd4900c1424d1  1rockstar62  \\\n",
       "0             0.0      0.0          0.0              0.0          0.0   \n",
       "1             0.0      0.0          0.0              0.0          0.0   \n",
       "2             0.0      0.0          0.0              0.0          0.0   \n",
       "3             0.0      0.0          0.0              0.0          0.0   \n",
       "4             0.0      0.0          0.0              0.0          0.0   \n",
       "\n",
       "   20skyhawkmm20  233liveonline  2for1lapdances  4tiles  4tp_kotc  ...  \\\n",
       "0            0.0            0.0             0.0     0.0       0.0  ...   \n",
       "1            0.0            0.0             0.0     0.0       0.0  ...   \n",
       "2            0.0            0.0             0.0     0.0       0.0  ...   \n",
       "3            0.0            0.0             0.0     0.0       0.0  ...   \n",
       "4            0.0            0.0             0.0     0.0       0.0  ...   \n",
       "\n",
       "   zaynmalik  zenande_mcfen  zeno001  zepp1978  zhenghxn  zojadelin  \\\n",
       "0        0.0            0.0      0.0       0.0       0.0        0.0   \n",
       "1        0.0            0.0      0.0       0.0       0.0        0.0   \n",
       "2        0.0            0.0      0.0       0.0       0.0        0.0   \n",
       "3        0.0            0.0      0.0       0.0       0.0        0.0   \n",
       "4        0.0            0.0      0.0       0.0       0.0        0.0   \n",
       "\n",
       "   zomatoaus  zourryart  zxathetis  id  \n",
       "0        0.0        0.0        0.0   0  \n",
       "1        0.0        0.0        0.0   2  \n",
       "2        0.0        0.0        0.0   3  \n",
       "3        0.0        0.0        0.0   9  \n",
       "4        0.0        0.0        0.0  11  \n",
       "\n",
       "[5 rows x 2311 columns]"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_tagged_index = test_tagged_users_categorical_features['tagged_users'].fillna('').index\n",
    "tagged_id = test_tagged_users_categorical_features['id'].fillna('')\n",
    "corpus    = test_tagged_users_categorical_features['tagged_users'].fillna('')\n",
    "\n",
    "#transform\n",
    "tf_idf_data = tf_idf_vec.transform(corpus)\n",
    "\n",
    "#create dataframe\n",
    "test_tf_idf_smooth_tagged=pd.DataFrame(tf_idf_data.toarray(),columns=tf_idf_vec.get_feature_names())\n",
    "test_tf_idf_smooth_tagged.set_index(test_tagged_index)\n",
    "test_tf_idf_smooth_tagged['id'] = tagged_id\n",
    "\n",
    "test_tf_idf_smooth_tagged.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Features HASHTAGS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "hashtags_numerical_features = df_text.copy()\n",
    "hashtags_numerical_features = hashtags_agregar_features_numericos(hashtags_numerical_features).drop(columns=['text'])\n",
    "hashtags_categorical_features = df_text.copy()\n",
    "hashtags_categorical_features = hashtags_agregar_features_categoricos(hashtags_categorical_features).drop(columns=['text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_hashtags_numerical_features = df_test_text.copy()\n",
    "test_hashtags_numerical_features = hashtags_agregar_features_numericos(test_hashtags_numerical_features).drop(columns=['text'])\n",
    "test_hashtags_categorical_features = df_test_text.copy()\n",
    "test_hashtags_categorical_features = hashtags_agregar_features_categoricos(test_hashtags_categorical_features).drop(columns=['text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>hashtags</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>[earthquake]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9</td>\n",
       "      <td>[Spokane, wildfires]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>11</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id              hashtags\n",
       "0   0                    []\n",
       "1   2          [earthquake]\n",
       "2   3                    []\n",
       "3   9  [Spokane, wildfires]\n",
       "4  11                    []"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_hashtags_categorical_features.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### TF-IDF - HASHTAGS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0215</th>\n",
       "      <th>034</th>\n",
       "      <th>039</th>\n",
       "      <th>06</th>\n",
       "      <th>09</th>\n",
       "      <th>1008pla_</th>\n",
       "      <th>1008planet_</th>\n",
       "      <th>124</th>\n",
       "      <th>140</th>\n",
       "      <th>16</th>\n",
       "      <th>...</th>\n",
       "      <th>yzf</th>\n",
       "      <th>zaatari</th>\n",
       "      <th>zimmerman</th>\n",
       "      <th>zionism</th>\n",
       "      <th>zionist</th>\n",
       "      <th>ziphimup</th>\n",
       "      <th>zippednews</th>\n",
       "      <th>zippoline</th>\n",
       "      <th>zombiefunrun2014</th>\n",
       "      <th>id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1909 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   0215  034  039   06   09  1008pla_  1008planet_  124  140   16  ...  yzf  \\\n",
       "0   0.0  0.0  0.0  0.0  0.0       0.0          0.0  0.0  0.0  0.0  ...  0.0   \n",
       "1   0.0  0.0  0.0  0.0  0.0       0.0          0.0  0.0  0.0  0.0  ...  0.0   \n",
       "2   0.0  0.0  0.0  0.0  0.0       0.0          0.0  0.0  0.0  0.0  ...  0.0   \n",
       "3   0.0  0.0  0.0  0.0  0.0       0.0          0.0  0.0  0.0  0.0  ...  0.0   \n",
       "4   0.0  0.0  0.0  0.0  0.0       0.0          0.0  0.0  0.0  0.0  ...  0.0   \n",
       "\n",
       "   zaatari  zimmerman  zionism  zionist  ziphimup  zippednews  zippoline  \\\n",
       "0      0.0        0.0      0.0      0.0       0.0         0.0        0.0   \n",
       "1      0.0        0.0      0.0      0.0       0.0         0.0        0.0   \n",
       "2      0.0        0.0      0.0      0.0       0.0         0.0        0.0   \n",
       "3      0.0        0.0      0.0      0.0       0.0         0.0        0.0   \n",
       "4      0.0        0.0      0.0      0.0       0.0         0.0        0.0   \n",
       "\n",
       "   zombiefunrun2014  id  \n",
       "0               0.0   1  \n",
       "1               0.0   4  \n",
       "2               0.0   5  \n",
       "3               0.0   6  \n",
       "4               0.0   7  \n",
       "\n",
       "[5 rows x 1909 columns]"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hashtags_categorical_features['hashtags'] = hashtags_categorical_features['hashtags'].apply(lambda x:toText(x))\n",
    "hashtag_index = hashtags_categorical_features['hashtags'].fillna('').index\n",
    "hashtag_id = hashtags_categorical_features['id'].fillna('')\n",
    "corpus = hashtags_categorical_features['hashtags'].fillna('')\n",
    "\n",
    "# Without Smoothing:\n",
    "tf_idf_vec = TfidfVectorizer(use_idf=True, \n",
    "                        smooth_idf=False,  \n",
    "                        ngram_range=(1,1),stop_words='english') # to use only  bigrams ngram_range=(2,2)\n",
    "#transform\n",
    "tf_idf_data = tf_idf_vec.fit_transform(corpus)\n",
    "\n",
    "#create dataframe\n",
    "df_tf_idf_hashtags=pd.DataFrame(tf_idf_data.toarray(),columns=tf_idf_vec.get_feature_names())\n",
    "df_tf_idf_hashtags.set_index(hashtag_index)\n",
    "df_tf_idf_hashtags['id'] = hashtag_id\n",
    "\n",
    "df_tf_idf_hashtags.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0215</th>\n",
       "      <th>034</th>\n",
       "      <th>039</th>\n",
       "      <th>06</th>\n",
       "      <th>09</th>\n",
       "      <th>1008pla_</th>\n",
       "      <th>1008planet_</th>\n",
       "      <th>124</th>\n",
       "      <th>140</th>\n",
       "      <th>16</th>\n",
       "      <th>...</th>\n",
       "      <th>yzf</th>\n",
       "      <th>zaatari</th>\n",
       "      <th>zimmerman</th>\n",
       "      <th>zionism</th>\n",
       "      <th>zionist</th>\n",
       "      <th>ziphimup</th>\n",
       "      <th>zippednews</th>\n",
       "      <th>zippoline</th>\n",
       "      <th>zombiefunrun2014</th>\n",
       "      <th>id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1909 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   0215  034  039   06   09  1008pla_  1008planet_  124  140   16  ...  yzf  \\\n",
       "0   0.0  0.0  0.0  0.0  0.0       0.0          0.0  0.0  0.0  0.0  ...  0.0   \n",
       "1   0.0  0.0  0.0  0.0  0.0       0.0          0.0  0.0  0.0  0.0  ...  0.0   \n",
       "2   0.0  0.0  0.0  0.0  0.0       0.0          0.0  0.0  0.0  0.0  ...  0.0   \n",
       "3   0.0  0.0  0.0  0.0  0.0       0.0          0.0  0.0  0.0  0.0  ...  0.0   \n",
       "4   0.0  0.0  0.0  0.0  0.0       0.0          0.0  0.0  0.0  0.0  ...  0.0   \n",
       "\n",
       "   zaatari  zimmerman  zionism  zionist  ziphimup  zippednews  zippoline  \\\n",
       "0      0.0        0.0      0.0      0.0       0.0         0.0        0.0   \n",
       "1      0.0        0.0      0.0      0.0       0.0         0.0        0.0   \n",
       "2      0.0        0.0      0.0      0.0       0.0         0.0        0.0   \n",
       "3      0.0        0.0      0.0      0.0       0.0         0.0        0.0   \n",
       "4      0.0        0.0      0.0      0.0       0.0         0.0        0.0   \n",
       "\n",
       "   zombiefunrun2014  id  \n",
       "0               0.0   0  \n",
       "1               0.0   2  \n",
       "2               0.0   3  \n",
       "3               0.0   9  \n",
       "4               0.0  11  \n",
       "\n",
       "[5 rows x 1909 columns]"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_hashtags_categorical_features['hashtags'] = test_hashtags_categorical_features['hashtags'].apply(lambda x:toText(x))\n",
    "hashtag_index = test_hashtags_categorical_features['hashtags'].fillna('').index\n",
    "hashtag_id = test_hashtags_categorical_features['id'].fillna('')\n",
    "corpus = test_hashtags_categorical_features['hashtags'].fillna('')\n",
    "\n",
    "#transform\n",
    "tf_idf_data = tf_idf_vec.transform(corpus)\n",
    "\n",
    "#create dataframe\n",
    "test_df_tf_idf_hashtags=pd.DataFrame(tf_idf_data.toarray(),columns=tf_idf_vec.get_feature_names())\n",
    "test_df_tf_idf_hashtags.set_index(hashtag_index)\n",
    "test_df_tf_idf_hashtags['id'] = hashtag_id\n",
    "\n",
    "test_df_tf_idf_hashtags.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0215</th>\n",
       "      <th>034</th>\n",
       "      <th>039</th>\n",
       "      <th>06</th>\n",
       "      <th>09</th>\n",
       "      <th>1008pla_</th>\n",
       "      <th>1008planet_</th>\n",
       "      <th>124</th>\n",
       "      <th>140</th>\n",
       "      <th>16</th>\n",
       "      <th>...</th>\n",
       "      <th>yzf</th>\n",
       "      <th>zaatari</th>\n",
       "      <th>zimmerman</th>\n",
       "      <th>zionism</th>\n",
       "      <th>zionist</th>\n",
       "      <th>ziphimup</th>\n",
       "      <th>zippednews</th>\n",
       "      <th>zippoline</th>\n",
       "      <th>zombiefunrun2014</th>\n",
       "      <th>id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1909 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   0215  034  039   06   09  1008pla_  1008planet_  124  140   16  ...  yzf  \\\n",
       "0   0.0  0.0  0.0  0.0  0.0       0.0          0.0  0.0  0.0  0.0  ...  0.0   \n",
       "1   0.0  0.0  0.0  0.0  0.0       0.0          0.0  0.0  0.0  0.0  ...  0.0   \n",
       "2   0.0  0.0  0.0  0.0  0.0       0.0          0.0  0.0  0.0  0.0  ...  0.0   \n",
       "3   0.0  0.0  0.0  0.0  0.0       0.0          0.0  0.0  0.0  0.0  ...  0.0   \n",
       "4   0.0  0.0  0.0  0.0  0.0       0.0          0.0  0.0  0.0  0.0  ...  0.0   \n",
       "\n",
       "   zaatari  zimmerman  zionism  zionist  ziphimup  zippednews  zippoline  \\\n",
       "0      0.0        0.0      0.0      0.0       0.0         0.0        0.0   \n",
       "1      0.0        0.0      0.0      0.0       0.0         0.0        0.0   \n",
       "2      0.0        0.0      0.0      0.0       0.0         0.0        0.0   \n",
       "3      0.0        0.0      0.0      0.0       0.0         0.0        0.0   \n",
       "4      0.0        0.0      0.0      0.0       0.0         0.0        0.0   \n",
       "\n",
       "   zombiefunrun2014  id  \n",
       "0               0.0   1  \n",
       "1               0.0   4  \n",
       "2               0.0   5  \n",
       "3               0.0   6  \n",
       "4               0.0   7  \n",
       "\n",
       "[5 rows x 1909 columns]"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hashtag_index = hashtags_categorical_features['hashtags'].fillna('').index\n",
    "hashtag_id = hashtags_categorical_features['id'].fillna('')\n",
    "corpus = hashtags_categorical_features['hashtags'].fillna('')\n",
    "\n",
    "# With Smoothing:\n",
    "\n",
    "tf_idf_vec = TfidfVectorizer(use_idf=True, \n",
    "                        smooth_idf=True,  \n",
    "                        ngram_range=(1,1),stop_words='english') # to use only  bigrams ngram_range=(2,2)\n",
    "#transform\n",
    "tf_idf_data = tf_idf_vec.fit_transform(corpus)\n",
    "\n",
    "#create dataframe\n",
    "df_tf_idf_smooth_hashtags=pd.DataFrame(tf_idf_data.toarray(),columns=tf_idf_vec.get_feature_names())\n",
    "df_tf_idf_smooth_hashtags.set_index(hashtag_index)\n",
    "df_tf_idf_smooth_hashtags['id'] = hashtag_id\n",
    "\n",
    "df_tf_idf_smooth_hashtags.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0215</th>\n",
       "      <th>034</th>\n",
       "      <th>039</th>\n",
       "      <th>06</th>\n",
       "      <th>09</th>\n",
       "      <th>1008pla_</th>\n",
       "      <th>1008planet_</th>\n",
       "      <th>124</th>\n",
       "      <th>140</th>\n",
       "      <th>16</th>\n",
       "      <th>...</th>\n",
       "      <th>yzf</th>\n",
       "      <th>zaatari</th>\n",
       "      <th>zimmerman</th>\n",
       "      <th>zionism</th>\n",
       "      <th>zionist</th>\n",
       "      <th>ziphimup</th>\n",
       "      <th>zippednews</th>\n",
       "      <th>zippoline</th>\n",
       "      <th>zombiefunrun2014</th>\n",
       "      <th>id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1909 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   0215  034  039   06   09  1008pla_  1008planet_  124  140   16  ...  yzf  \\\n",
       "0   0.0  0.0  0.0  0.0  0.0       0.0          0.0  0.0  0.0  0.0  ...  0.0   \n",
       "1   0.0  0.0  0.0  0.0  0.0       0.0          0.0  0.0  0.0  0.0  ...  0.0   \n",
       "2   0.0  0.0  0.0  0.0  0.0       0.0          0.0  0.0  0.0  0.0  ...  0.0   \n",
       "3   0.0  0.0  0.0  0.0  0.0       0.0          0.0  0.0  0.0  0.0  ...  0.0   \n",
       "4   0.0  0.0  0.0  0.0  0.0       0.0          0.0  0.0  0.0  0.0  ...  0.0   \n",
       "\n",
       "   zaatari  zimmerman  zionism  zionist  ziphimup  zippednews  zippoline  \\\n",
       "0      0.0        0.0      0.0      0.0       0.0         0.0        0.0   \n",
       "1      0.0        0.0      0.0      0.0       0.0         0.0        0.0   \n",
       "2      0.0        0.0      0.0      0.0       0.0         0.0        0.0   \n",
       "3      0.0        0.0      0.0      0.0       0.0         0.0        0.0   \n",
       "4      0.0        0.0      0.0      0.0       0.0         0.0        0.0   \n",
       "\n",
       "   zombiefunrun2014  id  \n",
       "0               0.0   0  \n",
       "1               0.0   2  \n",
       "2               0.0   3  \n",
       "3               0.0   9  \n",
       "4               0.0  11  \n",
       "\n",
       "[5 rows x 1909 columns]"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hashtag_index = test_hashtags_categorical_features['hashtags'].fillna('').index\n",
    "hashtag_id = test_hashtags_categorical_features['id'].fillna('')\n",
    "corpus = test_hashtags_categorical_features['hashtags'].fillna('')\n",
    "\n",
    "#transform\n",
    "tf_idf_data = tf_idf_vec.transform(corpus)\n",
    "\n",
    "#create dataframe\n",
    "test_df_tf_idf_smooth_hashtags=pd.DataFrame(tf_idf_data.toarray(),columns=tf_idf_vec.get_feature_names())\n",
    "test_df_tf_idf_smooth_hashtags.set_index(hashtag_index)\n",
    "test_df_tf_idf_smooth_hashtags['id'] = hashtag_id\n",
    "\n",
    "test_df_tf_idf_smooth_hashtags.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pronombres"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_pronoun(tweet, words):\n",
    "    res = \"\"\n",
    "    for w in tweet.lower().split(\" \"):\n",
    "        if w in words:\n",
    "            res+= \" \" + words[0]\n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 254,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_pronombres = train[['id']].copy()\n",
    "\n",
    "\n",
    "df_pronombres['she'] = train['text'].apply(lambda x: extract_pronoun(x, [\"she\", \"she's\"]))\n",
    "df_pronombres['he'] = train['text'].apply(lambda x: extract_pronoun(x, [\"he\", \"he's\"]))\n",
    "df_pronombres['it'] = train['text'].apply(lambda x: extract_pronoun(x, [\"it\", \"it's\"]))\n",
    "df_pronombres['i'] = train['text'].apply(lambda x: extract_pronoun(x, [\"i\", \"i'm\"]))\n",
    "df_pronombres['you'] = train['text'].apply(lambda x: extract_pronoun(x, [\"you\",\"you're\"]))\n",
    "df_pronombres['they'] = train['text'].apply(lambda x: extract_pronoun(x, [\"they\",\"they're\"]))\n",
    "df_pronombres['we'] = train['text'].apply(lambda x: extract_pronoun(x, [\"we\",\"we're\"]))\n",
    "\n",
    "df_pronombres['pronombres'] = df_pronombres['she'] + df_pronombres['he']+ df_pronombres['they']+\\\n",
    "            df_pronombres['it'] +df_pronombres['you'] + df_pronombres['i'] + df_pronombres['we'] \n",
    "df_pronombres['#pronombres'] = df_pronombres['pronombres'].apply(lambda x: len(x.split(\" \"))-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus = df_pronombres['pronombres']\n",
    "pronombres_id = df_pronombres['id'].fillna('')\n",
    "\n",
    "# Without Smoothing:\n",
    "tf_idf_vec = TfidfVectorizer(use_idf=True, \n",
    "                        smooth_idf=False,  \n",
    "                        ngram_range=(1,1),stop_words=[]) # to use only  bigrams ngram_range=(2,2)\n",
    "#transform\n",
    "tf_idf_data = tf_idf_vec.fit_transform(corpus)\n",
    "\n",
    "#create dataframe\n",
    "pronombres=pd.DataFrame(tf_idf_data.toarray(),columns=tf_idf_vec.get_feature_names())\n",
    "pronombres['id'] = pronombres_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 255,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_pronombres_test = test[['id']].copy()\n",
    "\n",
    "\n",
    "df_pronombres_test['she'] = test['text'].apply(lambda x: extract_pronoun(x, [\"she\", \"she's\"]))\n",
    "df_pronombres_test['he'] = test['text'].apply(lambda x: extract_pronoun(x, [\"he\", \"he's\"]))\n",
    "df_pronombres_test['it'] = test['text'].apply(lambda x: extract_pronoun(x, [\"it\", \"it's\"]))\n",
    "df_pronombres_test['i'] = test['text'].apply(lambda x: extract_pronoun(x, [\"i\", \"i'm\"]))\n",
    "df_pronombres_test['you'] = test['text'].apply(lambda x: extract_pronoun(x, [\"you\",\"you're\"]))\n",
    "df_pronombres_test['they'] = test['text'].apply(lambda x: extract_pronoun(x, [\"they\",\"they're\"]))\n",
    "df_pronombres_test['we'] = test['text'].apply(lambda x: extract_pronoun(x, [\"we\",\"we're\"]))\n",
    "\n",
    "df_pronombres_test['pronombres'] = df_pronombres_test['she'] + df_pronombres_test['he']+ df_pronombres_test['they']+\\\n",
    "            df_pronombres_test['it'] +df_pronombres_test['you'] + df_pronombres_test['i'] + df_pronombres_test['we']\n",
    "\n",
    "df_pronombres_test['#pronombres'] = df_pronombres_test['pronombres'].apply(lambda x: len(x.split(\" \"))-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus = df_pronombres_test['pronombres']\n",
    "pronombres_id = df_pronombres['id'].fillna('')\n",
    "\n",
    "# Without Smoothing:\n",
    "tf_idf_vec = TfidfVectorizer(use_idf=True, \n",
    "                        smooth_idf=False,  \n",
    "                        ngram_range=(1,1),stop_words=[]) # to use only  bigrams ngram_range=(2,2)\n",
    "#transform\n",
    "tf_idf_data = tf_idf_vec.fit_transform(corpus)\n",
    "\n",
    "#create dataframe\n",
    "pronombres_test=pd.DataFrame(tf_idf_data.toarray(),columns=tf_idf_vec.get_feature_names())\n",
    "pronombres_test['id'] = pronombres_id"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### pos tag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk import word_tokenize, pos_tag\n",
    "\n",
    "def get_pos_tag(sentence):\n",
    "    res = \"\"\n",
    "    pos_tags =  pos_tag(word_tokenize(sentence))\n",
    "    for tag in pos_tags:\n",
    "        res+=tag[1]+\" \"\n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_pos_tag = train[['id']].copy()\n",
    "\n",
    "df_pos_tag['pos_tag'] = train['text'].apply(lambda x: get_pos_tag(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>pos_tag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>PRP$ NNS VBP DT NNP IN DT # NN NNP NNP NNP PRP...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>NNP NN IN NNP NNP NNP . NNP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>DT NNS VBD TO VB IN NN '' VBP VBG VBN IN NNS ....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>CD NNS JJ # NNS VBP NNS IN NNP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7</td>\n",
       "      <td>RB VBN VBD DT NN IN NNP # NNP IN NN IN # NNS V...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id                                            pos_tag\n",
       "0   1  PRP$ NNS VBP DT NNP IN DT # NN NNP NNP NNP PRP...\n",
       "1   4                       NNP NN IN NNP NNP NNP . NNP \n",
       "2   5  DT NNS VBD TO VB IN NN '' VBP VBG VBN IN NNS ....\n",
       "3   6                    CD NNS JJ # NNS VBP NNS IN NNP \n",
       "4   7  RB VBN VBD DT NN IN NNP # NNP IN NN IN # NNS V..."
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_pos_tag.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus = df_pos_tag['pos_tag']\n",
    "pos_tag_id = df_pos_tag['id'].fillna('')\n",
    "\n",
    "# Without Smoothing:\n",
    "tf_idf_vec = TfidfVectorizer(use_idf=True, \n",
    "                        smooth_idf=False,  \n",
    "                        ngram_range=(1,1),stop_words=[]) # to use only  bigrams ngram_range=(2,2)\n",
    "#transform\n",
    "tf_idf_data = tf_idf_vec.fit_transform(corpus)\n",
    "\n",
    "#create dataframe\n",
    "pos_tag_train=pd.DataFrame(tf_idf_data.toarray(),columns=tf_idf_vec.get_feature_names())\n",
    "pos_tag_train['id'] = pos_tag_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_pos_tag_test = test[['id']].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_pos_tag_test['pos_tag'] = test['text'].apply(lambda x: get_pos_tag(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus = df_pos_tag_test['pos_tag']\n",
    "pos_tag_id = df_pos_tag_test['id'].fillna('')\n",
    "\n",
    "# Without Smoothing:\n",
    "tf_idf_vec = TfidfVectorizer(use_idf=True, \n",
    "                        smooth_idf=False,  \n",
    "                        ngram_range=(1,1),stop_words=[]) # to use only  bigrams ngram_range=(2,2)\n",
    "#transform\n",
    "tf_idf_data = tf_idf_vec.fit_transform(corpus)\n",
    "\n",
    "#create dataframe\n",
    "pos_tag_test = pd.DataFrame(tf_idf_data.toarray(),columns=tf_idf_vec.get_feature_names())\n",
    "pos_tag_test['id'] = pos_tag_id"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Exportacion de Dfs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_tf_idf_tagged.to_csv('features/train/features_tagged.csv', encoding='utf-8')\n",
    "#df_tf_idf_smooth_tagged.to_csv('features/train/features_tagged_smooth.csv', encoding='utf-8')\n",
    "#df_tf_idf_hashtags.to_csv('features/train/features_hashtags.csv', encoding='utf-8')\n",
    "#df_tf_idf_smooth_hashtags.to_csv('features/train/features_hashtags_smooth.csv', encoding='utf-8')\n",
    "#tagged_users_numerical_features.to_csv('features/train/features_arrobas_numerical.csv', encoding='utf-8')\n",
    "#text_numerical_features.to_csv('features/train/features_text_numerical.csv', encoding='utf-8')\n",
    "#hashtags_numerical_features.to_csv('features/train/features_hashtags_numerical.csv', encoding='utf-8')\n",
    "# pronombres.to_csv('features/train/features_pronombres.csv', encoding='utf-8')\n",
    "#pos_tag_train.to_csv('features/train/feature_pos_tag.csv', encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "#test_tf_idf_tagged.to_csv('features/test/features_tagged.csv', encoding='utf-8')\n",
    "#test_tf_idf_smooth_tagged.to_csv('features/test/features_tagged_smooth.csv', encoding='utf-8')\n",
    "#test_df_tf_idf_hashtags.to_csv('features/test/features_hashtags.csv', encoding='utf-8')\n",
    "#test_df_tf_idf_smooth_hashtags.to_csv('features/test/features_hashtags_smooth.csv', encoding='utf-8')\n",
    "#test_tagged_users_numerical_features.to_csv('features/test/features_arrobas_numerical.csv', encoding='utf-8')\n",
    "#test_text_numerical_features.to_csv('features/test/features_text_numerical.csv', encoding='utf-8')\n",
    "#test_hashtags_numerical_features.to_csv('features/test/features_hashtags_numerical.csv', encoding='utf-8')\n",
    "#pronombres_test.to_csv('features/test/features_pronombres.csv', encoding='utf-8')\n",
    "#pos_tag_test.to_csv('features/test/feature_pos_tag.csv', encoding='utf-8')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.3 Location"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "def agregar_features_location_in_text(df):\n",
    "    df[\"cities\"]       = df[\"text\"].apply(lambda x: GeoText(str(x)).cities)\n",
    "    df[\"cities_str\"]   = df[\"cities\"].apply(lambda x: to_string(x,\",\"))\n",
    "    df[\"country\"]      = df[\"text\"].apply(lambda x: get_country(x))\n",
    "    df[\"counties\"]     = df[\"text\"].apply(lambda x: check_if_US_county(str(x)))\n",
    "    df[\"counties_str\"] = df[\"counties\"].apply(lambda x: to_string(x,\",\"))\n",
    "    df[\"states\"]       = df[\"text\"].apply(lambda x: check_if_US_state(str(x)))\n",
    "    \n",
    "    df.loc[(df['states'].str.len() == 0), 'states'] = df.counties.apply(lambda x: get_US_state(x))\n",
    "\n",
    "    df[\"country_str\"]  = df[\"country\"].apply(lambda x: to_string(x,\",\"))\n",
    "    df[\"states_str\"]   = df[\"states\"].apply(lambda x: to_string(x,\",\"))\n",
    "      \n",
    "    df.loc[~pd.isnull(df['counties_str']), 'country_str'] = 'US'\n",
    "    df.loc[~pd.isnull(df['states_str']), 'country_str']   = 'US'\n",
    "\n",
    "    df[\"continent\"]    = df[\"text\"].apply(lambda x: check_if_continent(str(x)))\n",
    "    df.loc[df['continent'].str.len() == 0, 'continent'] = df.country_str.apply(lambda x: get_continent(x))\n",
    "\n",
    "    df[\"continent_str\"] = df[\"continent\"].apply(lambda x: to_string(x,\",\"))\n",
    "    df[\"has_country\"]   = (np.where(pd.isnull(df['country_str']), False, True)).astype(int)\n",
    "    df[\"has_city\"]      = (np.where(pd.isnull(df['cities_str']), False, True)).astype(int)\n",
    "    df[\"has_county\"]    = (np.where(pd.isnull(df['counties_str']), False, True)).astype(int)\n",
    "    df[\"has_state\"]     = (np.where(pd.isnull(df['states_str']), False, True)).astype(int)\n",
    "    df[\"has_continent\"] = (np.where(pd.isnull(df['continent_str']), False, True)).astype(int)\n",
    "    return df\n",
    "\n",
    "def agregar_features_location(df):\n",
    "    df[\"has_location\"] = (df[\"location\"] != \"\").astype(int)\n",
    "    df[\"cities\"]       = df[\"location\"].apply(lambda x: GeoText(str(x)).cities)\n",
    "    df[\"cities_str\"]   = df[\"cities\"].apply(lambda x: to_string(x,\",\"))\n",
    "    df[\"country\"]      = df[\"location\"].apply(lambda x: get_country(x))\n",
    "    df[\"counties\"]     = df[\"location\"].apply(lambda x: check_if_US_county(str(x)))\n",
    "    df[\"counties_str\"] = df[\"counties\"].apply(lambda x: to_string(x,\",\"))\n",
    "    df[\"states\"]       = df[\"location\"].apply(lambda x: check_if_US_state(str(x)))\n",
    "    df.loc[(df['states'].str.len() == 0), 'states'] = df.counties.apply(lambda x: get_US_state(x))\n",
    "    df[\"country_str\"]  = df[\"country\"].apply(lambda x: to_string(x,\",\"))\n",
    "    df[\"states_str\"]   = df[\"states\"].apply(lambda x: to_string(x,\",\"))\n",
    "    df.loc[~pd.isnull(df['counties_str']), 'country_str'] = 'US'\n",
    "    df.loc[~pd.isnull(df['states_str']), 'country_str']   = 'US'\n",
    "    df[\"continent\"]    = df[\"location\"].apply(lambda x: check_if_continent(str(x)))\n",
    "    df.loc[df['continent'].str.len() == 0, 'continent']   = df.country_str.apply(lambda x: get_continent(x))\n",
    "    df[\"continent_str\"] = df[\"continent\"].apply(lambda x: to_string(x,\",\"))\n",
    "    df[\"has_country\"]   = (np.where(pd.isnull(df['country_str']), False, True)).astype(int)\n",
    "    df[\"has_city\"]      = (np.where(pd.isnull(df['cities_str']), False, True)).astype(int)\n",
    "    df[\"has_county\"]    = (np.where(pd.isnull(df['counties_str']), False, True)).astype(int)\n",
    "    df[\"has_state\"]     = (np.where(pd.isnull(df['states_str']), False, True)).astype(int)\n",
    "    df[\"has_continent\"] = (np.where(pd.isnull(df['continent_str']), False, True)).astype(int)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Features LOCATION IN TEXT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/anichu/Programs/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:55: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n"
     ]
    }
   ],
   "source": [
    "train_location_in_text = train[[\"id\", \"text\"]].copy()\n",
    "train_location_in_text['text'] = train_location_in_text['text'].apply(lambda x: limpiar_tweet_location(x))\n",
    "train_location_in_text = agregar_features_location_in_text(train_location_in_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/anichu/Programs/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:55: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n"
     ]
    }
   ],
   "source": [
    "test_location_in_text = test[[\"id\", \"text\"]].copy()\n",
    "test_location_in_text['text'] = test_location_in_text['text'].apply(lambda x: limpiar_tweet_location(x))\n",
    "test_location_in_text = agregar_features_location_in_text(test_location_in_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/anichu/Programs/anaconda3/lib/python3.6/site-packages/pandas/core/indexing.py:1494: FutureWarning: \n",
      "Passing list-likes to .loc or [] with any missing label will raise\n",
      "KeyError in the future, you can use .reindex() as an alternative.\n",
      "\n",
      "See the documentation here:\n",
      "https://pandas.pydata.org/pandas-docs/stable/indexing.html#deprecate-loc-reindex-listlike\n",
      "  return self._getitem_tuple(key)\n"
     ]
    }
   ],
   "source": [
    "train_location_in_text_numerical = train_location_in_text.loc[:,[\"id\",\"target\",\"has_country\",\"has_city\",\"has_county\",\"has_state\",\"has_continent\"]]\n",
    "test_location_in_text_numerical  = test_location_in_text.loc[:,[\"id\",\"has_country\",\"has_city\",\"has_county\",\"has_state\",\"has_continent\"]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Encoding de variables categóricas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/anichu/Programs/anaconda3/lib/python3.6/site-packages/sklearn/preprocessing/_label.py:987: UserWarning: unknown class(es) ['AL', 'ER', 'FJ', 'GH', 'GY', 'HT', 'KR', 'MV', 'QA', 'SG', 'SL', 'TL', 'TM'] will be ignored\n",
      "  .format(sorted(unknown, key=str)))\n"
     ]
    }
   ],
   "source": [
    "location_in_text = train_location_in_text.copy()\n",
    "train_location_in_text_country = location_in_text.loc[:,['id','country']].join(pd.DataFrame(mlb.fit_transform(location_in_text.pop('country')),\n",
    "                          columns=mlb.classes_,\n",
    "                          index=location_in_text.index).add_prefix('country_')).drop(columns=['country'])\n",
    "\n",
    "location_in_text_test = test_location_in_text.copy()\n",
    "test_location_in_text_country = location_in_text_test.loc[:,['id','country']].join(pd.DataFrame(mlb.transform(location_in_text_test.pop('country')),\n",
    "                          columns=mlb.classes_,\n",
    "                          index=location_in_text_test.index).add_prefix('country_')).drop(columns=['country'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/anichu/Programs/anaconda3/lib/python3.6/site-packages/sklearn/preprocessing/_label.py:987: UserWarning: unknown class(es) ['Agen', 'Amsterdam', 'Asia', 'Bell', 'Bethesda', 'Boo', 'Cape Town', 'Clinton', 'Coney Island', 'Cornwall', 'Dome', 'Fall River', 'Forest Grove', 'Foster', 'Gaza', 'Gresham', 'Grove', 'Hale', 'Harare', 'Hyderabad', 'Isla Vista', 'Issaquah', 'Johnson', 'Kano', 'Keller', 'Kolkata', 'Liberty', 'Liverpool', 'Long Island', 'Male', 'March', 'Marion', 'Mary', 'Newcastle', 'Raleigh', 'Rossendale', 'Same', 'Sterling', 'West Raleigh'] will be ignored\n",
      "  .format(sorted(unknown, key=str)))\n"
     ]
    }
   ],
   "source": [
    "#location_in_text = train_location_in_text.copy()\n",
    "train_location_in_text_city = location_in_text.loc[:,['id','cities']].join(pd.DataFrame(mlb.fit_transform(location_in_text.pop('cities')),\n",
    "                          columns=mlb.classes_,\n",
    "                          index=location_in_text.index).add_prefix('city_')).drop(columns=['cities'])\n",
    "\n",
    "test_location_in_text_city = location_in_text_test.loc[:,['id','cities']].join(pd.DataFrame(mlb.transform(location_in_text_test.pop('cities')),\n",
    "                          columns=mlb.classes_,\n",
    "                          index=location_in_text_test.index).add_prefix('city_')).drop(columns=['cities'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/anichu/Programs/anaconda3/lib/python3.6/site-packages/sklearn/preprocessing/_label.py:987: UserWarning: unknown class(es) ['Avery County', 'Burlington County', 'Jackson County'] will be ignored\n",
      "  .format(sorted(unknown, key=str)))\n"
     ]
    }
   ],
   "source": [
    "#location_in_text = train_location_in_text.copy()\n",
    "train_location_in_text_county = location_in_text.loc[:,['id','counties']].join(pd.DataFrame(mlb.fit_transform(location_in_text.pop('counties')),\n",
    "                          columns=mlb.classes_,\n",
    "                          index=location_in_text.index).add_prefix('county_')).drop(columns=['counties'])\n",
    "\n",
    "test_location_in_text_county = location_in_text_test.loc[:,['id','counties']].join(pd.DataFrame(mlb.transform(location_in_text_test.pop('counties')),\n",
    "                          columns=mlb.classes_,\n",
    "                          index=location_in_text_test.index).add_prefix('county_')).drop(columns=['counties'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "#location_in_text = train_location_in_text.copy()\n",
    "train_location_in_text_state = location_in_text.loc[:,['id','states']].join(pd.DataFrame(mlb.fit_transform(location_in_text.pop('states')),\n",
    "                          columns=mlb.classes_,\n",
    "                          index=location_in_text.index).add_prefix('state_')).drop(columns=['states'])\n",
    "\n",
    "test_location_in_text_state = location_in_text_test.loc[:,['id','states']].join(pd.DataFrame(mlb.transform(location_in_text_test.pop('states')),\n",
    "                          columns=mlb.classes_,\n",
    "                          index=location_in_text_test.index).add_prefix('state_')).drop(columns=['states'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "#location_in_text = train_location_in_text.copy()\n",
    "train_location_in_text_continent = location_in_text.loc[:,['id','continent']].join(pd.DataFrame(mlb.fit_transform(location_in_text.pop('continent')),\n",
    "                          columns=mlb.classes_,\n",
    "                          index=location_in_text.index).add_prefix('continent_')).drop(columns=['continent'])\n",
    "\n",
    "test_location_in_text_continent = location_in_text_test.loc[:,['id','continent']].join(pd.DataFrame(mlb.transform(location_in_text_test.pop('continent')),\n",
    "                          columns=mlb.classes_,\n",
    "                          index=location_in_text_test.index).add_prefix('continent_')).drop(columns=['continent'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Exportación dataframes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"\\ntrain_location_in_text_numerical.to_csv('features/train/features_location_in_text_numerical.csv', encoding='utf-8')\\ntest_location_in_text_numerical.to_csv('features/test/features_location_in_text_numerical.csv', encoding='utf-8')\\ntrain_location_in_text_country.to_csv('features/train/features_country_in_text_encoded.csv', encoding='utf-8')\\ntest_location_in_text_country.to_csv('features/test/features_country_in_text_encoded.csv', encoding='utf-8')\\ntrain_location_in_text_city.to_csv('features/train/features_city_in_text_encoded.csv', encoding='utf-8')\\ntest_location_in_text_city.to_csv('features/test/features_city_in_text_encoded.csv', encoding='utf-8')\\ntrain_location_in_text_county.to_csv('features/train/features_county_in_text_encoded.csv', encoding='utf-8')\\ntest_location_in_text_county.to_csv('features/test/features_county_in_text_encoded.csv', encoding='utf-8')\\ntrain_location_in_text_state.to_csv('features/train/features_state_in_text_encoded.csv', encoding='utf-8')\\ntest_location_in_text_state.to_csv('features/test/features_state_in_text_encoded.csv', encoding='utf-8')\\ntrain_location_in_text_continent.to_csv('features/train/features_continent_in_text_encoded.csv', encoding='utf-8')\\ntest_location_in_text_continent.to_csv('features/test/features_continent_in_text_encoded.csv', encoding='utf-8')\\n\""
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#EXPORTAR CSV\n",
    "'''\n",
    "train_location_in_text_numerical.to_csv('features/train/features_location_in_text_numerical.csv', encoding='utf-8')\n",
    "test_location_in_text_numerical.to_csv('features/test/features_location_in_text_numerical.csv', encoding='utf-8')\n",
    "train_location_in_text_country.to_csv('features/train/features_country_in_text_encoded.csv', encoding='utf-8')\n",
    "test_location_in_text_country.to_csv('features/test/features_country_in_text_encoded.csv', encoding='utf-8')\n",
    "train_location_in_text_city.to_csv('features/train/features_city_in_text_encoded.csv', encoding='utf-8')\n",
    "test_location_in_text_city.to_csv('features/test/features_city_in_text_encoded.csv', encoding='utf-8')\n",
    "train_location_in_text_county.to_csv('features/train/features_county_in_text_encoded.csv', encoding='utf-8')\n",
    "test_location_in_text_county.to_csv('features/test/features_county_in_text_encoded.csv', encoding='utf-8')\n",
    "train_location_in_text_state.to_csv('features/train/features_state_in_text_encoded.csv', encoding='utf-8')\n",
    "test_location_in_text_state.to_csv('features/test/features_state_in_text_encoded.csv', encoding='utf-8')\n",
    "train_location_in_text_continent.to_csv('features/train/features_continent_in_text_encoded.csv', encoding='utf-8')\n",
    "test_location_in_text_continent.to_csv('features/test/features_continent_in_text_encoded.csv', encoding='utf-8')\n",
    "'''"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Features LOCATION"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_location = train[[\"id\", \"location\",\"target\"]].copy().fillna(\"\")\n",
    "train_location[\"location\"] = train_location[\"location\"].apply(lambda x: limpiar_tweet_location(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_location = test[[\"id\", \"location\"]].copy().fillna(\"\")\n",
    "test_location[\"location\"] = test_location[\"location\"].apply(lambda x: limpiar_tweet_location(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/anichu/Programs/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:55: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n"
     ]
    }
   ],
   "source": [
    "train_location = agregar_features_location(train_location)\n",
    "test_location = agregar_features_location(test_location)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_location_numerical = train_location.loc[:,[\"id\", \"target\",\"has_country\",\"has_city\",\"has_county\",\"has_state\",\"has_continent\"]]\n",
    "test_location_numerical  = test_location.loc[:,[\"id\",\"has_country\",\"has_city\",\"has_county\",\"has_state\",\"has_continent\"]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Encoding de variables categóricas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/anichu/Programs/anaconda3/lib/python3.6/site-packages/sklearn/preprocessing/_label.py:987: UserWarning: unknown class(es) ['BS', 'GY', 'KP', 'LS', 'MM', 'MT', 'NP', 'PA', 'TN', 'YE'] will be ignored\n",
      "  .format(sorted(unknown, key=str)))\n"
     ]
    }
   ],
   "source": [
    "location = train_location.copy()\n",
    "train_location_country = location.loc[:,['id','country']].join(pd.DataFrame(mlb.fit_transform(location.pop('country')),\n",
    "                          columns=mlb.classes_,\n",
    "                          index=location.index).add_prefix('country_')).drop(columns=['country'])\n",
    "\n",
    "location_test = test_location.copy()\n",
    "test_location_country = location_test.loc[:,['id','country']].join(pd.DataFrame(mlb.transform(location_test.pop('country')),\n",
    "                          columns=mlb.classes_,\n",
    "                          index=location_test.index).add_prefix('country_')).drop(columns=['country'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/anichu/Programs/anaconda3/lib/python3.6/site-packages/sklearn/preprocessing/_label.py:987: UserWarning: unknown class(es) ['Algonquin', 'Amman', 'Ankeny', 'Balikpapan', 'Barnsley', 'Bellingham', 'Berkeley', 'Boca Raton', 'Bowling Green', 'Brussels', 'Cairo', 'Calabasas', 'Cape Coral', 'Carrboro', 'Carthage', 'Cary', 'Cedar City', 'Cheltenham', 'Chillicothe', 'Clearlake', 'Conway', 'Courtenay', 'Dalton', 'Davis', 'Deerfield Beach', 'Edmonton', 'Edwardsville', 'Eugene', 'Fano', 'Fargo', 'Fayetteville', 'Florence', 'Forest Grove', 'Fredericksburg', 'Fredericton', 'Goldsboro', 'Green', 'Greer', 'Gresham', 'Gurgaon', 'Hannover', 'Harare', 'Hartford', 'Highlands Ranch', 'Homewood', 'Hudson', 'Hull', 'Huntington Beach', 'Hutchinson', 'Indian Trail', 'Irvine', 'Issaquah', 'Jabalpur', 'Jefferson', 'Johnstown', 'Kearney', 'Keffi', 'Lake Charles', 'Lake Worth', 'Lakeland', 'Lansing', 'Laporte', 'Lawrence', 'Lehigh Acres', 'Lemon Grove', 'Lille', 'Lucknow', 'Mechanicsville', 'Midland', 'Milano', 'Mississauga', 'Missoula', 'Mobile', 'Modesto', 'Mombasa', 'Moncton', 'Monroe', 'Morganton', 'Morgantown', 'Muscat', 'Muskegon', 'Nanaimo', 'Newburgh', 'Newport', 'Normal', 'North Bellmore', 'North Hollywood', 'North Tonawanda', 'Norton', 'Okinawa', 'Ottawa', 'Panama City', 'Pasadena', 'Patna', 'Phuket', 'Pico Rivera', 'Piscataway', 'Pleasantville', 'Poughkeepsie', 'Pune', 'Rahway', 'Redding', 'Rexburg', 'Roermond', 'Rowlett', 'Runcorn', 'Saint Andrews', 'Saint John', 'Saint Petersburg', 'Salem', 'Sittwe', 'South Boston', 'Springfield', 'Stamford', 'Staten Island', 'Stockport', 'Sudbury', 'Sunderland', 'Superior', 'Surrey', 'Swansea', 'Syracuse', 'Temple', 'Tinley Park', 'Townsville', 'Turin', 'Twin Falls', 'Udaipur', 'University', 'Vineland', 'Virginia Beach', 'Wayne', 'West Covina', 'Wickford', 'Williamsport', 'Winchester', 'Windsor', 'Woodbridge', 'Yakima'] will be ignored\n",
      "  .format(sorted(unknown, key=str)))\n"
     ]
    }
   ],
   "source": [
    "#location_in_text = train_location_in_text.copy()\n",
    "train_location_city = location.loc[:,['id','cities']].join(pd.DataFrame(mlb.fit_transform(location.pop('cities')),\n",
    "                          columns=mlb.classes_,\n",
    "                          index=location.index).add_prefix('city_')).drop(columns=['cities'])\n",
    "\n",
    "test_location_city = location_test.loc[:,['id','cities']].join(pd.DataFrame(mlb.transform(location_test.pop('cities')),\n",
    "                          columns=mlb.classes_,\n",
    "                          index=location_test.index).add_prefix('city_')).drop(columns=['cities'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/anichu/Programs/anaconda3/lib/python3.6/site-packages/sklearn/preprocessing/_label.py:987: UserWarning: unknown class(es) ['Dade County', 'Johnson County', 'Ventura County'] will be ignored\n",
      "  .format(sorted(unknown, key=str)))\n"
     ]
    }
   ],
   "source": [
    "#location_in_text = train_location_in_text.copy()\n",
    "train_location_county = location.loc[:,['id','counties']].join(pd.DataFrame(mlb.fit_transform(location.pop('counties')),\n",
    "                          columns=mlb.classes_,\n",
    "                          index=location.index).add_prefix('county_')).drop(columns=['counties'])\n",
    "\n",
    "test_location_county = location_test.loc[:,['id','counties']].join(pd.DataFrame(mlb.transform(location_test.pop('counties')),\n",
    "                          columns=mlb.classes_,\n",
    "                          index=location_test.index).add_prefix('county_')).drop(columns=['counties'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "#location_in_text = train_location_in_text.copy()\n",
    "train_location_state = location.loc[:,['id','states']].join(pd.DataFrame(mlb.fit_transform(location.pop('states')),\n",
    "                          columns=mlb.classes_,\n",
    "                          index=location.index).add_prefix('state_')).drop(columns=['states'])\n",
    "\n",
    "test_location_state = location_test.loc[:,['id','states']].join(pd.DataFrame(mlb.transform(location_test.pop('states')),\n",
    "                          columns=mlb.classes_,\n",
    "                          index=location_test.index).add_prefix('state_')).drop(columns=['states'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_location_continent = location.loc[:,['id','continent']].join(pd.DataFrame(mlb.fit_transform(location.pop('continent')),\n",
    "                          columns=mlb.classes_,\n",
    "                          index=location.index).add_prefix('continent_')).drop(columns=['continent'])\n",
    "\n",
    "test_location_continent = location_test.loc[:,['id','continent']].join(pd.DataFrame(mlb.transform(location_test.pop('continent')),\n",
    "                          columns=mlb.classes_,\n",
    "                          index=location_test.index).add_prefix('continent_')).drop(columns=['continent'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"train_location_numerical.to_csv('features/train/features_location_numerical.csv', encoding='utf-8')\\ntest_location_numerical.to_csv('features/test/features_location_numerical.csv', encoding='utf-8')\\ntrain_location_country.to_csv('features/train/features_country_encoded.csv', encoding='utf-8')\\ntest_location_country.to_csv('features/test/features_country_encoded.csv', encoding='utf-8')\\ntrain_location_city.to_csv('features/train/features_city_encoded.csv', encoding='utf-8')\\ntest_location_city.to_csv('features/test/features_city_encoded.csv', encoding='utf-8')\\ntrain_location_county.to_csv('features/train/features_county_encoded.csv', encoding='utf-8')\\ntest_location_county.to_csv('features/test/features_county_encoded.csv', encoding='utf-8')\\ntrain_location_state.to_csv('features/train/features_state_encoded.csv', encoding='utf-8')\\ntest_location_state.to_csv('features/test/features_state_encoded.csv', encoding='utf-8')\\ntrain_location_continent.to_csv('features/train/features_continent_encoded.csv', encoding='utf-8')\\ntest_location_continent.to_csv('features/test/features_continent_encoded.csv', encoding='utf-8')\""
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#EXPORTAR CSV\n",
    "'''train_location_numerical.to_csv('features/train/features_location_numerical.csv', encoding='utf-8')\n",
    "test_location_numerical.to_csv('features/test/features_location_numerical.csv', encoding='utf-8')\n",
    "train_location_country.to_csv('features/train/features_country_encoded.csv', encoding='utf-8')\n",
    "test_location_country.to_csv('features/test/features_country_encoded.csv', encoding='utf-8')\n",
    "train_location_city.to_csv('features/train/features_city_encoded.csv', encoding='utf-8')\n",
    "test_location_city.to_csv('features/test/features_city_encoded.csv', encoding='utf-8')\n",
    "train_location_county.to_csv('features/train/features_county_encoded.csv', encoding='utf-8')\n",
    "test_location_county.to_csv('features/test/features_county_encoded.csv', encoding='utf-8')\n",
    "train_location_state.to_csv('features/train/features_state_encoded.csv', encoding='utf-8')\n",
    "test_location_state.to_csv('features/test/features_state_encoded.csv', encoding='utf-8')\n",
    "train_location_continent.to_csv('features/train/features_continent_encoded.csv', encoding='utf-8')\n",
    "test_location_continent.to_csv('features/test/features_continent_encoded.csv', encoding='utf-8')'''"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Location label countries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Para aquellos tweets con más de dos paises, separo en columnas\n",
    "\n",
    "countries = location['country_str'].str.split(\",\", expand = True)\n",
    "\n",
    "countries[\"id\"] = location[\"id\"]\n",
    "countries.set_index(\"id\")\n",
    "\n",
    "countries1 = pd.get_dummies(countries[0],\"Country\")\n",
    "countries2 = pd.get_dummies(countries[1],\"Country\")\n",
    "\n",
    "# Uno las columnas\n",
    "for country in countries1.columns:\n",
    "    if(country in countries2.columns):\n",
    "        countries1[country] = countries1[country] + countries2[country]\n",
    "for country in countries2.columns:\n",
    "    if(country in countries1.columns):\n",
    "        countries1[country] = countries2[country]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Country_AE</th>\n",
       "      <th>Country_AF</th>\n",
       "      <th>Country_AQ</th>\n",
       "      <th>Country_AR</th>\n",
       "      <th>Country_AT</th>\n",
       "      <th>Country_AU</th>\n",
       "      <th>Country_BB</th>\n",
       "      <th>Country_BD</th>\n",
       "      <th>Country_BE</th>\n",
       "      <th>Country_BG</th>\n",
       "      <th>...</th>\n",
       "      <th>Country_TH</th>\n",
       "      <th>Country_TM</th>\n",
       "      <th>Country_UA</th>\n",
       "      <th>Country_UG</th>\n",
       "      <th>Country_US</th>\n",
       "      <th>Country_UY</th>\n",
       "      <th>Country_VE</th>\n",
       "      <th>Country_VN</th>\n",
       "      <th>Country_ZA</th>\n",
       "      <th>Country_ZW</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 101 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Country_AE  Country_AF  Country_AQ  Country_AR  Country_AT  Country_AU  \\\n",
       "0           0           0           0           0           0           0   \n",
       "1           0           0           0           0           0           0   \n",
       "\n",
       "   Country_BB  Country_BD  Country_BE  Country_BG  ...  Country_TH  \\\n",
       "0           0           0           0           0  ...           0   \n",
       "1           0           0           0           0  ...           0   \n",
       "\n",
       "   Country_TM  Country_UA  Country_UG  Country_US  Country_UY  Country_VE  \\\n",
       "0           0           0           0           0           0           0   \n",
       "1           0           0           0           0           0           0   \n",
       "\n",
       "   Country_VN  Country_ZA  Country_ZW  \n",
       "0           0           0           0  \n",
       "1           0           0           0  \n",
       "\n",
       "[2 rows x 101 columns]"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "countries1.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### BOW: Coutry"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "#bow_country_index = location['country_str'].fillna('').index\n",
    "corpus = list(location['country_str'].fillna(''))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "country_vectorizer = CountVectorizer()\n",
    "\n",
    "bow_country = country_vectorizer.fit_transform(corpus).todense()\n",
    "df_bow_country = pd.DataFrame(bow_country)\n",
    "df_bow_country = df_bow_country.set_index(location['id'].index)\n",
    "df_bow_country.columns = ['country_'+ c for c in  country_vectorizer.get_feature_names()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>country_ae</th>\n",
       "      <th>country_af</th>\n",
       "      <th>country_aq</th>\n",
       "      <th>country_ar</th>\n",
       "      <th>country_at</th>\n",
       "      <th>country_au</th>\n",
       "      <th>country_bb</th>\n",
       "      <th>country_bd</th>\n",
       "      <th>country_be</th>\n",
       "      <th>country_bg</th>\n",
       "      <th>...</th>\n",
       "      <th>country_th</th>\n",
       "      <th>country_tm</th>\n",
       "      <th>country_ua</th>\n",
       "      <th>country_ug</th>\n",
       "      <th>country_us</th>\n",
       "      <th>country_uy</th>\n",
       "      <th>country_ve</th>\n",
       "      <th>country_vn</th>\n",
       "      <th>country_za</th>\n",
       "      <th>country_zw</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 102 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   country_ae  country_af  country_aq  country_ar  country_at  country_au  \\\n",
       "0           0           0           0           0           0           0   \n",
       "1           0           0           0           0           0           0   \n",
       "2           0           0           0           0           0           0   \n",
       "3           0           0           0           0           0           0   \n",
       "4           0           0           0           0           0           0   \n",
       "\n",
       "   country_bb  country_bd  country_be  country_bg  ...  country_th  \\\n",
       "0           0           0           0           0  ...           0   \n",
       "1           0           0           0           0  ...           0   \n",
       "2           0           0           0           0  ...           0   \n",
       "3           0           0           0           0  ...           0   \n",
       "4           0           0           0           0  ...           0   \n",
       "\n",
       "   country_tm  country_ua  country_ug  country_us  country_uy  country_ve  \\\n",
       "0           0           0           0           0           0           0   \n",
       "1           0           0           0           0           0           0   \n",
       "2           0           0           0           0           0           0   \n",
       "3           0           0           0           0           0           0   \n",
       "4           0           0           0           0           0           0   \n",
       "\n",
       "   country_vn  country_za  country_zw  \n",
       "0           0           0           0  \n",
       "1           0           0           0  \n",
       "2           0           0           0  \n",
       "3           0           0           0  \n",
       "4           0           0           0  \n",
       "\n",
       "[5 rows x 102 columns]"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_bow_country.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tf-Idf: Country"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>country_ae</th>\n",
       "      <th>country_af</th>\n",
       "      <th>country_aq</th>\n",
       "      <th>country_ar</th>\n",
       "      <th>country_au</th>\n",
       "      <th>country_bb</th>\n",
       "      <th>country_bd</th>\n",
       "      <th>country_bg</th>\n",
       "      <th>country_bh</th>\n",
       "      <th>country_br</th>\n",
       "      <th>...</th>\n",
       "      <th>country_sy</th>\n",
       "      <th>country_th</th>\n",
       "      <th>country_tm</th>\n",
       "      <th>country_ua</th>\n",
       "      <th>country_ug</th>\n",
       "      <th>country_uy</th>\n",
       "      <th>country_ve</th>\n",
       "      <th>country_vn</th>\n",
       "      <th>country_za</th>\n",
       "      <th>country_zw</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 89 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   country_ae  country_af  country_aq  country_ar  country_au  country_bb  \\\n",
       "0         0.0         0.0         0.0         0.0         0.0         0.0   \n",
       "1         0.0         0.0         0.0         0.0         0.0         0.0   \n",
       "2         0.0         0.0         0.0         0.0         0.0         0.0   \n",
       "3         0.0         0.0         0.0         0.0         0.0         0.0   \n",
       "4         0.0         0.0         0.0         0.0         0.0         0.0   \n",
       "\n",
       "   country_bd  country_bg  country_bh  country_br  ...  country_sy  \\\n",
       "0         0.0         0.0         0.0         0.0  ...         0.0   \n",
       "1         0.0         0.0         0.0         0.0  ...         0.0   \n",
       "2         0.0         0.0         0.0         0.0  ...         0.0   \n",
       "3         0.0         0.0         0.0         0.0  ...         0.0   \n",
       "4         0.0         0.0         0.0         0.0  ...         0.0   \n",
       "\n",
       "   country_th  country_tm  country_ua  country_ug  country_uy  country_ve  \\\n",
       "0         0.0         0.0         0.0         0.0         0.0         0.0   \n",
       "1         0.0         0.0         0.0         0.0         0.0         0.0   \n",
       "2         0.0         0.0         0.0         0.0         0.0         0.0   \n",
       "3         0.0         0.0         0.0         0.0         0.0         0.0   \n",
       "4         0.0         0.0         0.0         0.0         0.0         0.0   \n",
       "\n",
       "   country_vn  country_za  country_zw  \n",
       "0         0.0         0.0         0.0  \n",
       "1         0.0         0.0         0.0  \n",
       "2         0.0         0.0         0.0  \n",
       "3         0.0         0.0         0.0  \n",
       "4         0.0         0.0         0.0  \n",
       "\n",
       "[5 rows x 89 columns]"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Without Smoothing:\n",
    "\n",
    "tf_idf_vec = TfidfVectorizer(use_idf=True, \n",
    "                        smooth_idf=False,  \n",
    "                        ngram_range=(1,1),stop_words='english') # to use only  bigrams ngram_range=(2,2)\n",
    "#transform\n",
    "tf_idf_data = tf_idf_vec.fit_transform(corpus)\n",
    "\n",
    "#create dataframe\n",
    "df_tf_idf_country=pd.DataFrame(tf_idf_data.toarray(),columns=tf_idf_vec.get_feature_names())\n",
    "df_tf_idf_country.columns = ['country_'+ c for c in df_tf_idf_country.columns]\n",
    "df_tf_idf_country = df_tf_idf_country.set_index(location['id'].index)\n",
    "\n",
    "df_tf_idf_country.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>country_ae</th>\n",
       "      <th>country_af</th>\n",
       "      <th>country_aq</th>\n",
       "      <th>country_ar</th>\n",
       "      <th>country_au</th>\n",
       "      <th>country_bb</th>\n",
       "      <th>country_bd</th>\n",
       "      <th>country_bg</th>\n",
       "      <th>country_bh</th>\n",
       "      <th>country_br</th>\n",
       "      <th>...</th>\n",
       "      <th>country_sy</th>\n",
       "      <th>country_th</th>\n",
       "      <th>country_tm</th>\n",
       "      <th>country_ua</th>\n",
       "      <th>country_ug</th>\n",
       "      <th>country_uy</th>\n",
       "      <th>country_ve</th>\n",
       "      <th>country_vn</th>\n",
       "      <th>country_za</th>\n",
       "      <th>country_zw</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 89 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   country_ae  country_af  country_aq  country_ar  country_au  country_bb  \\\n",
       "0         0.0         0.0         0.0         0.0         0.0         0.0   \n",
       "1         0.0         0.0         0.0         0.0         0.0         0.0   \n",
       "2         0.0         0.0         0.0         0.0         0.0         0.0   \n",
       "3         0.0         0.0         0.0         0.0         0.0         0.0   \n",
       "4         0.0         0.0         0.0         0.0         0.0         0.0   \n",
       "\n",
       "   country_bd  country_bg  country_bh  country_br  ...  country_sy  \\\n",
       "0         0.0         0.0         0.0         0.0  ...         0.0   \n",
       "1         0.0         0.0         0.0         0.0  ...         0.0   \n",
       "2         0.0         0.0         0.0         0.0  ...         0.0   \n",
       "3         0.0         0.0         0.0         0.0  ...         0.0   \n",
       "4         0.0         0.0         0.0         0.0  ...         0.0   \n",
       "\n",
       "   country_th  country_tm  country_ua  country_ug  country_uy  country_ve  \\\n",
       "0         0.0         0.0         0.0         0.0         0.0         0.0   \n",
       "1         0.0         0.0         0.0         0.0         0.0         0.0   \n",
       "2         0.0         0.0         0.0         0.0         0.0         0.0   \n",
       "3         0.0         0.0         0.0         0.0         0.0         0.0   \n",
       "4         0.0         0.0         0.0         0.0         0.0         0.0   \n",
       "\n",
       "   country_vn  country_za  country_zw  \n",
       "0         0.0         0.0         0.0  \n",
       "1         0.0         0.0         0.0  \n",
       "2         0.0         0.0         0.0  \n",
       "3         0.0         0.0         0.0  \n",
       "4         0.0         0.0         0.0  \n",
       "\n",
       "[5 rows x 89 columns]"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# With Smoothing:\n",
    "\n",
    "tf_idf_vec = TfidfVectorizer(use_idf=True, \n",
    "                        smooth_idf=True,  \n",
    "                        ngram_range=(1,1),stop_words='english') # to use only  bigrams ngram_range=(2,2)\n",
    "#transform\n",
    "tf_idf_data = tf_idf_vec.fit_transform(corpus)\n",
    "\n",
    "#create dataframe\n",
    "column_names_c = ['country_'+ c for c in  tf_idf_vec.get_feature_names()]\n",
    "df_tf_idf_smooth_country=pd.DataFrame(tf_idf_data.toarray(),columns=column_names_c)\n",
    "df_tf_idf_smooth_country.set_index(location['id'].index)\n",
    "\n",
    "df_tf_idf_smooth_country.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "nueva_posicion= df_bow_country.columns.get_loc('country_ae')\n",
    "df_bow_country.insert(loc= nueva_posicion, column='id', value = df_bow_country.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_tf_idf_smooth_country.insert(loc= nueva_posicion, column='id', value = df_tf_idf_smooth_country.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "# DOWNLOAD CSV LOCATION\n",
    "\n",
    "# TRAIN\n",
    "\n",
    "#df_tf_idf_smooth_country.to_csv('features/train/features_location_tf_idf.csv', encoding='utf-8')\n",
    "#df_bow_country.to_csv('features/train/features_location_bow.csv', encoding='utf-8')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.4 Links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "def agregar_features_links(df):\n",
    "\n",
    "    df[\"links_domain\"] = df[\"http\"].apply(lambda x: get_dominio(x))\n",
    "    df[\"links_encription\"] = df[\"http\"].apply(lambda x: type_http(x))\n",
    "    df[\"links_cant\"] = df[\"links_cant\"]\n",
    "    df[\"links_encription_type\"] = df[\"links_encription\"].apply(lambda x: link_encription_generalize(x))\n",
    "    df[\"cant_failed_links\"] = df[\"http\"].apply(lambda x: count_None(x))\n",
    "    df[\"links_domain_str\"] = df[\"links_domain\"].apply(lambda x: to_string(x,\",\"))\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "links_train = agregar_features_links(links_train)\n",
    "links_test = agregar_features_links(links_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/anichu/Programs/anaconda3/lib/python3.6/site-packages/pandas/core/indexing.py:1494: FutureWarning: \n",
      "Passing list-likes to .loc or [] with any missing label will raise\n",
      "KeyError in the future, you can use .reindex() as an alternative.\n",
      "\n",
      "See the documentation here:\n",
      "https://pandas.pydata.org/pandas-docs/stable/indexing.html#deprecate-loc-reindex-listlike\n",
      "  return self._getitem_tuple(key)\n"
     ]
    }
   ],
   "source": [
    "features_links_train_numerical = links_train.loc[:,['target','id','links_cant','cant_failed_links']]\n",
    "features_links_test_numerical = links_test.loc[:,['target','id','links_cant','cant_failed_links']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dominios train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "domain_train_index = links_train['links_domain_str'].dropna().index\n",
    "corpus = list(links_train['links_domain_str'].dropna())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### BOW"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>domain_10news</th>\n",
       "      <th>domain_1stheadlines</th>\n",
       "      <th>domain_24x7freeadvice</th>\n",
       "      <th>domain_28chronicle</th>\n",
       "      <th>domain_28national</th>\n",
       "      <th>domain_28ndtv</th>\n",
       "      <th>domain_28zero</th>\n",
       "      <th>domain_29</th>\n",
       "      <th>domain_2c</th>\n",
       "      <th>...</th>\n",
       "      <th>domain_zalkus</th>\n",
       "      <th>domain_zamanalwsl</th>\n",
       "      <th>domain_zeenews</th>\n",
       "      <th>domain_zenit</th>\n",
       "      <th>domain_zeppy</th>\n",
       "      <th>domain_zero</th>\n",
       "      <th>domain_zerohedge</th>\n",
       "      <th>domain_ziprecruiter</th>\n",
       "      <th>domain_zoma</th>\n",
       "      <th>domain_zw</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>48</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>49</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>50</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>53</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>55</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1079 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  domain_10news  domain_1stheadlines  domain_24x7freeadvice  \\\n",
       "0  48              0                    0                      0   \n",
       "1  49              0                    0                      0   \n",
       "2  50              0                    0                      0   \n",
       "3  53              0                    0                      0   \n",
       "4  55              0                    0                      0   \n",
       "\n",
       "   domain_28chronicle  domain_28national  domain_28ndtv  domain_28zero  \\\n",
       "0                   0                  0              0              0   \n",
       "1                   0                  0              0              0   \n",
       "2                   0                  0              0              0   \n",
       "3                   0                  0              0              0   \n",
       "4                   0                  0              0              0   \n",
       "\n",
       "   domain_29  domain_2c  ...  domain_zalkus  domain_zamanalwsl  \\\n",
       "0          0          0  ...              0                  0   \n",
       "1          0          0  ...              0                  0   \n",
       "2          0          0  ...              0                  0   \n",
       "3          0          0  ...              0                  0   \n",
       "4          0          0  ...              0                  0   \n",
       "\n",
       "   domain_zeenews  domain_zenit  domain_zeppy  domain_zero  domain_zerohedge  \\\n",
       "0               0             0             0            0                 0   \n",
       "1               0             0             0            0                 0   \n",
       "2               0             0             0            0                 0   \n",
       "3               0             0             0            0                 0   \n",
       "4               0             0             0            0                 0   \n",
       "\n",
       "   domain_ziprecruiter  domain_zoma  domain_zw  \n",
       "0                    0            0          0  \n",
       "1                    0            0          0  \n",
       "2                    0            0          0  \n",
       "3                    0            0          0  \n",
       "4                    0            0          0  \n",
       "\n",
       "[5 rows x 1079 columns]"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "domain_vectorizer = CountVectorizer(analyzer = \"word\",   \\\n",
    "                             tokenizer = None,    \\\n",
    "                             preprocessor = None, \\\n",
    "                             stop_words = [\"www\", \"com\"],   \\\n",
    "                             max_features = 5000)\n",
    "\n",
    "bow_domain_train = domain_vectorizer.fit_transform(corpus).todense()\n",
    "df_bow_domain_train = pd.DataFrame(bow_domain_train)\n",
    "df_bow_domain_train = df_bow_domain_train.set_index(domain_train_index)\n",
    "df_bow_domain_train.columns = [\"domain_\" + c for c in domain_vectorizer.get_feature_names()]\n",
    "df_bow_domain_train = df_bow_domain_train.reset_index()\n",
    "\n",
    "df_bow_domain_train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Tf-Idf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>domain_10news</th>\n",
       "      <th>domain_1stheadlines</th>\n",
       "      <th>domain_24x7freeadvice</th>\n",
       "      <th>domain_28chronicle</th>\n",
       "      <th>domain_28national</th>\n",
       "      <th>domain_28ndtv</th>\n",
       "      <th>domain_28zero</th>\n",
       "      <th>domain_29</th>\n",
       "      <th>domain_2c</th>\n",
       "      <th>...</th>\n",
       "      <th>domain_zalkus</th>\n",
       "      <th>domain_zamanalwsl</th>\n",
       "      <th>domain_zeenews</th>\n",
       "      <th>domain_zenit</th>\n",
       "      <th>domain_zeppy</th>\n",
       "      <th>domain_zero</th>\n",
       "      <th>domain_zerohedge</th>\n",
       "      <th>domain_ziprecruiter</th>\n",
       "      <th>domain_zoma</th>\n",
       "      <th>domain_zw</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>48</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>49</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>50</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>53</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>55</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1057 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  domain_10news  domain_1stheadlines  domain_24x7freeadvice  \\\n",
       "0  48            0.0                  0.0                    0.0   \n",
       "1  49            0.0                  0.0                    0.0   \n",
       "2  50            0.0                  0.0                    0.0   \n",
       "3  53            0.0                  0.0                    0.0   \n",
       "4  55            0.0                  0.0                    0.0   \n",
       "\n",
       "   domain_28chronicle  domain_28national  domain_28ndtv  domain_28zero  \\\n",
       "0                 0.0                0.0            0.0            0.0   \n",
       "1                 0.0                0.0            0.0            0.0   \n",
       "2                 0.0                0.0            0.0            0.0   \n",
       "3                 0.0                0.0            0.0            0.0   \n",
       "4                 0.0                0.0            0.0            0.0   \n",
       "\n",
       "   domain_29  domain_2c  ...  domain_zalkus  domain_zamanalwsl  \\\n",
       "0        0.0        0.0  ...            0.0                0.0   \n",
       "1        0.0        0.0  ...            0.0                0.0   \n",
       "2        0.0        0.0  ...            0.0                0.0   \n",
       "3        0.0        0.0  ...            0.0                0.0   \n",
       "4        0.0        0.0  ...            0.0                0.0   \n",
       "\n",
       "   domain_zeenews  domain_zenit  domain_zeppy  domain_zero  domain_zerohedge  \\\n",
       "0             0.0           0.0           0.0          0.0               0.0   \n",
       "1             0.0           0.0           0.0          0.0               0.0   \n",
       "2             0.0           0.0           0.0          0.0               0.0   \n",
       "3             0.0           0.0           0.0          0.0               0.0   \n",
       "4             0.0           0.0           0.0          0.0               0.0   \n",
       "\n",
       "   domain_ziprecruiter  domain_zoma  domain_zw  \n",
       "0                  0.0          0.0        0.0  \n",
       "1                  0.0          0.0        0.0  \n",
       "2                  0.0          0.0        0.0  \n",
       "3                  0.0          0.0        0.0  \n",
       "4                  0.0          0.0        0.0  \n",
       "\n",
       "[5 rows x 1057 columns]"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Without Smoothing:\n",
    "\n",
    "tf_idf_vec = TfidfVectorizer(use_idf=True, \n",
    "                        smooth_idf=False,  \n",
    "                        ngram_range=(1,1),stop_words='english') # to use only  bigrams ngram_range=(2,2)\n",
    "#transform\n",
    "tf_idf_data = tf_idf_vec.fit_transform(corpus)\n",
    "\n",
    "#create dataframe\n",
    "\n",
    "df_tf_idf_domain_train_cols = [\"domain_\" + c for c in tf_idf_vec.get_feature_names()]\n",
    "df_tf_idf_domain_train=pd.DataFrame(tf_idf_data.toarray(),columns=df_tf_idf_domain_train_cols)\n",
    "df_tf_idf_domain_train = df_tf_idf_domain_train.set_index(domain_train_index)\n",
    "df_tf_idf_domain_train = df_tf_idf_domain_train.reset_index()\n",
    "\n",
    "del df_tf_idf_domain_train[\"domain_www\"]\n",
    "del df_tf_idf_domain_train[\"domain_com\"]\n",
    "\n",
    "df_tf_idf_domain_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>domain_10news</th>\n",
       "      <th>domain_1stheadlines</th>\n",
       "      <th>domain_24x7freeadvice</th>\n",
       "      <th>domain_28chronicle</th>\n",
       "      <th>domain_28national</th>\n",
       "      <th>domain_28ndtv</th>\n",
       "      <th>domain_28zero</th>\n",
       "      <th>domain_29</th>\n",
       "      <th>domain_2c</th>\n",
       "      <th>...</th>\n",
       "      <th>domain_zalkus</th>\n",
       "      <th>domain_zamanalwsl</th>\n",
       "      <th>domain_zeenews</th>\n",
       "      <th>domain_zenit</th>\n",
       "      <th>domain_zeppy</th>\n",
       "      <th>domain_zero</th>\n",
       "      <th>domain_zerohedge</th>\n",
       "      <th>domain_ziprecruiter</th>\n",
       "      <th>domain_zoma</th>\n",
       "      <th>domain_zw</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>48</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>49</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>50</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>53</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>55</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1057 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  domain_10news  domain_1stheadlines  domain_24x7freeadvice  \\\n",
       "0  48            0.0                  0.0                    0.0   \n",
       "1  49            0.0                  0.0                    0.0   \n",
       "2  50            0.0                  0.0                    0.0   \n",
       "3  53            0.0                  0.0                    0.0   \n",
       "4  55            0.0                  0.0                    0.0   \n",
       "\n",
       "   domain_28chronicle  domain_28national  domain_28ndtv  domain_28zero  \\\n",
       "0                 0.0                0.0            0.0            0.0   \n",
       "1                 0.0                0.0            0.0            0.0   \n",
       "2                 0.0                0.0            0.0            0.0   \n",
       "3                 0.0                0.0            0.0            0.0   \n",
       "4                 0.0                0.0            0.0            0.0   \n",
       "\n",
       "   domain_29  domain_2c  ...  domain_zalkus  domain_zamanalwsl  \\\n",
       "0        0.0        0.0  ...            0.0                0.0   \n",
       "1        0.0        0.0  ...            0.0                0.0   \n",
       "2        0.0        0.0  ...            0.0                0.0   \n",
       "3        0.0        0.0  ...            0.0                0.0   \n",
       "4        0.0        0.0  ...            0.0                0.0   \n",
       "\n",
       "   domain_zeenews  domain_zenit  domain_zeppy  domain_zero  domain_zerohedge  \\\n",
       "0             0.0           0.0           0.0          0.0               0.0   \n",
       "1             0.0           0.0           0.0          0.0               0.0   \n",
       "2             0.0           0.0           0.0          0.0               0.0   \n",
       "3             0.0           0.0           0.0          0.0               0.0   \n",
       "4             0.0           0.0           0.0          0.0               0.0   \n",
       "\n",
       "   domain_ziprecruiter  domain_zoma  domain_zw  \n",
       "0                  0.0          0.0        0.0  \n",
       "1                  0.0          0.0        0.0  \n",
       "2                  0.0          0.0        0.0  \n",
       "3                  0.0          0.0        0.0  \n",
       "4                  0.0          0.0        0.0  \n",
       "\n",
       "[5 rows x 1057 columns]"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# With Smoothing:\n",
    "\n",
    "tf_idf_vec_smooth = TfidfVectorizer(use_idf=True, \n",
    "                        smooth_idf=True,  \n",
    "                        ngram_range=(1,1),stop_words='english') # to use only  bigrams ngram_range=(2,2)\n",
    "#transform\n",
    "tf_idf_data = tf_idf_vec_smooth.fit_transform(corpus)\n",
    "\n",
    "#create dataframe\n",
    "\n",
    "df_tf_idf_smooth_domain_train_cols = [\"domain_\" + c for c in tf_idf_vec_smooth.get_feature_names()]\n",
    "df_tf_idf_smooth_domain_train=pd.DataFrame(tf_idf_data.toarray(),columns=df_tf_idf_domain_train_cols)\n",
    "df_tf_idf_smooth_domain_train = df_tf_idf_smooth_domain_train.set_index(domain_train_index)\n",
    "df_tf_idf_smooth_domain_train = df_tf_idf_smooth_domain_train.reset_index()\n",
    "\n",
    "del df_tf_idf_smooth_domain_train[\"domain_www\"]\n",
    "del df_tf_idf_smooth_domain_train[\"domain_com\"]\n",
    "\n",
    "df_tf_idf_smooth_domain_train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dominios test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "domain_test_index = links_test['links_domain_str'].dropna().index\n",
    "corpus = list(links_test['links_domain_str'].dropna())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### BOW"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>domain_10news</th>\n",
       "      <th>domain_1stheadlines</th>\n",
       "      <th>domain_24x7freeadvice</th>\n",
       "      <th>domain_28chronicle</th>\n",
       "      <th>domain_28national</th>\n",
       "      <th>domain_28ndtv</th>\n",
       "      <th>domain_28zero</th>\n",
       "      <th>domain_29</th>\n",
       "      <th>domain_2c</th>\n",
       "      <th>...</th>\n",
       "      <th>domain_zalkus</th>\n",
       "      <th>domain_zamanalwsl</th>\n",
       "      <th>domain_zeenews</th>\n",
       "      <th>domain_zenit</th>\n",
       "      <th>domain_zeppy</th>\n",
       "      <th>domain_zero</th>\n",
       "      <th>domain_zerohedge</th>\n",
       "      <th>domain_ziprecruiter</th>\n",
       "      <th>domain_zoma</th>\n",
       "      <th>domain_zw</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>46</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>58</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>70</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>75</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>84</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1079 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  domain_10news  domain_1stheadlines  domain_24x7freeadvice  \\\n",
       "0  46              0                    0                      0   \n",
       "1  58              0                    0                      0   \n",
       "2  70              0                    0                      0   \n",
       "3  75              0                    0                      0   \n",
       "4  84              0                    0                      0   \n",
       "\n",
       "   domain_28chronicle  domain_28national  domain_28ndtv  domain_28zero  \\\n",
       "0                   0                  0              0              0   \n",
       "1                   0                  0              0              0   \n",
       "2                   0                  0              0              0   \n",
       "3                   0                  0              0              0   \n",
       "4                   0                  0              0              0   \n",
       "\n",
       "   domain_29  domain_2c  ...  domain_zalkus  domain_zamanalwsl  \\\n",
       "0          0          0  ...              0                  0   \n",
       "1          0          0  ...              0                  0   \n",
       "2          0          0  ...              0                  0   \n",
       "3          0          0  ...              0                  0   \n",
       "4          0          0  ...              0                  0   \n",
       "\n",
       "   domain_zeenews  domain_zenit  domain_zeppy  domain_zero  domain_zerohedge  \\\n",
       "0               0             0             0            0                 0   \n",
       "1               0             0             0            0                 0   \n",
       "2               0             0             0            0                 0   \n",
       "3               0             0             0            0                 0   \n",
       "4               0             0             0            0                 0   \n",
       "\n",
       "   domain_ziprecruiter  domain_zoma  domain_zw  \n",
       "0                    0            0          0  \n",
       "1                    0            0          0  \n",
       "2                    0            0          0  \n",
       "3                    0            0          0  \n",
       "4                    0            0          0  \n",
       "\n",
       "[5 rows x 1079 columns]"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bow_domain_test = domain_vectorizer.transform(corpus).todense()\n",
    "df_bow_domain_test = pd.DataFrame(bow_domain_test)\n",
    "df_bow_domain_test = df_bow_domain_test.set_index(domain_test_index)\n",
    "df_bow_domain_test.columns = [\"domain_\" + c for c in domain_vectorizer.get_feature_names()]\n",
    "df_bow_domain_test = df_bow_domain_test.reset_index()\n",
    "\n",
    "df_bow_domain_test.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Tf-Idf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "#transform\n",
    "tf_idf_data = tf_idf_vec.transform(corpus)\n",
    "\n",
    "#create dataframe\n",
    "df_tf_idf_domain_test_cols = [\"domain_\" + c for c in tf_idf_vec.get_feature_names()]\n",
    "df_tf_idf_domain_test = pd.DataFrame(tf_idf_data.toarray(),columns=df_tf_idf_domain_test_cols)\n",
    "df_tf_idf_domain_test = df_tf_idf_domain_test.set_index(domain_test_index)\n",
    "df_tf_idf_domain_test = df_tf_idf_domain_test.reset_index()\n",
    "\n",
    "del df_tf_idf_domain_test[\"domain_www\"]\n",
    "del df_tf_idf_domain_test[\"domain_com\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "#transform\n",
    "tf_idf_data = tf_idf_vec_smooth.transform(corpus)\n",
    "\n",
    "#create dataframe\n",
    "df_tf_idf_smooth_domain_test_cols = [\"domain_\" + c for c in tf_idf_vec_smooth.get_feature_names()]\n",
    "df_tf_idf_smooth_domain_test = pd.DataFrame(tf_idf_data.toarray(),columns=df_tf_idf_domain_test_cols)\n",
    "df_tf_idf_smooth_domain_test = df_tf_idf_smooth_domain_test.set_index(domain_test_index)\n",
    "df_tf_idf_smooth_domain_test = df_tf_idf_smooth_domain_test.reset_index()\n",
    "\n",
    "del df_tf_idf_smooth_domain_test[\"domain_www\"]\n",
    "del df_tf_idf_smooth_domain_test[\"domain_com\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'features_links_train_numerical.to_csv(\"features/train/features_links_numerical.csv\", encoding=\\'utf-8\\')\\ndf_tf_idf_smooth_domain_train.to_csv(\\'features/train/features_domain_smooth_tf_idf.csv\\', encoding=\\'utf-8\\')\\ndf_tf_idf_domain_train.to_csv(\\'features/train/features_domain_tf_idf.csv\\', encoding=\\'utf-8\\')\\ndf_bow_domain_train.to_csv(\\'features/train/features_domain_bow.csv\\', encoding=\\'utf-8\\')\\n\\n\\n# TEST\\n\\nfeatures_links_test_numerical.to_csv(\"features/test/features_links_numerical.csv\", encoding=\\'utf-8\\')\\ndf_tf_idf_smooth_domain_test.to_csv(\\'features/test/features_domain_smooth_tf_idf.csv\\', encoding=\\'utf-8\\')\\ndf_tf_idf_domain_test.to_csv(\\'features/test/features_domain_tf_idf.csv\\', encoding=\\'utf-8\\')\\ndf_bow_domain_test.to_csv(\\'features/test/features_domain_bow.csv\\', encoding=\\'utf-8\\')'"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# DOWNLOAD CSV LINKS\n",
    "\n",
    "# TRAIN\n",
    "\n",
    "'''features_links_train_numerical.to_csv(\"features/train/features_links_numerical.csv\", encoding='utf-8')\n",
    "df_tf_idf_smooth_domain_train.to_csv('features/train/features_domain_smooth_tf_idf.csv', encoding='utf-8')\n",
    "df_tf_idf_domain_train.to_csv('features/train/features_domain_tf_idf.csv', encoding='utf-8')\n",
    "df_bow_domain_train.to_csv('features/train/features_domain_bow.csv', encoding='utf-8')\n",
    "\n",
    "\n",
    "# TEST\n",
    "\n",
    "features_links_test_numerical.to_csv(\"features/test/features_links_numerical.csv\", encoding='utf-8')\n",
    "df_tf_idf_smooth_domain_test.to_csv('features/test/features_domain_smooth_tf_idf.csv', encoding='utf-8')\n",
    "df_tf_idf_domain_test.to_csv('features/test/features_domain_tf_idf.csv', encoding='utf-8')\n",
    "df_bow_domain_test.to_csv('features/test/features_domain_bow.csv', encoding='utf-8')'''"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.5 Tags"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Informacion general tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creo el dataframe\n",
    "train_tags = train[['id','text','target']].copy(deep=True)\n",
    "\n",
    "# Limpio el dataframe\n",
    "train_tags['new'] = train_tags[\"text\"].apply(lambda x: clean_text_strict(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creo el dataframe\n",
    "test_tags = test[['id','text']].copy(deep=True)\n",
    "\n",
    "# Limpio el dataframe\n",
    "test_tags['new'] = test_tags[\"text\"].apply(lambda x: clean_text_strict(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "#  Eliminacion de terminos, caracteres y stopwords innecesarios para generar los tags\n",
    "#convierte de list a string\n",
    "train_tags['new'] = train_tags['new'].apply(lambda x: [item for item in x.split() if item not in stopwordsList])\n",
    "train_tags.new = train_tags.new.apply(prettify)\n",
    "\n",
    "test_tags['new'] = test_tags['new'].apply(lambda x: [item for item in x.split() if item not in stopwordsList])\n",
    "test_tags.new = test_tags.new.apply(prettify)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "w_tokenizer = nltk.tokenize.WhitespaceTokenizer()\n",
    "lemmatizer = nltk.stem.WordNetLemmatizer()\n",
    "\n",
    "train_tags['new_tokenized'] = train_tags.new.apply(lemmatize_text2)\n",
    "train_tags.new_tokenized = train_tags.new_tokenized.apply(prettify)\n",
    "\n",
    "test_tags['new_tokenized'] = test_tags.new.apply(lemmatize_text2)\n",
    "test_tags.new_tokenized = test_tags.new_tokenized.apply(prettify)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = train_tags['new_tokenized'].apply(getText).str.lower().str.cat(sep=' ')\n",
    "\n",
    "words = nltk.tokenize.word_tokenize(a)\n",
    "word_dist = nltk.FreqDist(words)\n",
    "\n",
    "this_col = pd.Series(['word','frequency'])\n",
    "\n",
    "rslt = pd.DataFrame(word_dist.most_common(400),columns = this_col)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "frequent_words = rslt ['word']\n",
    "unique_frequent_words = set(frequent_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>target</th>\n",
       "      <th>text</th>\n",
       "      <th>tags</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Our Deeds are the Reason of this #earthquake M...</td>\n",
       "      <td>[reason, earthquake]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>Forest fire near La Ronge Sask. Canada</td>\n",
       "      <td>[forest, fire]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>All residents asked to 'shelter in place' are ...</td>\n",
       "      <td>[evacuation, officer]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>13,000 people receive #wildfires evacuation or...</td>\n",
       "      <td>[evacuation, california, people, wildfire]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>Just got sent this photo from Ruby #Alaska as ...</td>\n",
       "      <td>[smoke, photo, wildfire]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  target                                               text  \\\n",
       "0   1       1  Our Deeds are the Reason of this #earthquake M...   \n",
       "1   4       1             Forest fire near La Ronge Sask. Canada   \n",
       "2   5       1  All residents asked to 'shelter in place' are ...   \n",
       "3   6       1  13,000 people receive #wildfires evacuation or...   \n",
       "4   7       1  Just got sent this photo from Ruby #Alaska as ...   \n",
       "\n",
       "                                         tags  \n",
       "0                        [reason, earthquake]  \n",
       "1                              [forest, fire]  \n",
       "2                       [evacuation, officer]  \n",
       "3  [evacuation, california, people, wildfire]  \n",
       "4                    [smoke, photo, wildfire]  "
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_tags['tags'] = train_tags.new_tokenized.apply(word_finder)\n",
    "train_info = train_tags[['id','target','text','tags']]\n",
    "\n",
    "train_info.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = test_tags['new_tokenized'].apply(getText).str.lower().str.cat(sep=' ')\n",
    "\n",
    "words = nltk.tokenize.word_tokenize(a)\n",
    "word_dist = nltk.FreqDist(words)\n",
    "\n",
    "this_col = pd.Series(['word','frequency'])\n",
    "\n",
    "rslt = pd.DataFrame(word_dist.most_common(400),columns = this_col)\n",
    "\n",
    "frequent_words = rslt ['word']\n",
    "unique_frequent_words = set(frequent_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>text</th>\n",
       "      <th>tags</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Just happened a terrible car crash</td>\n",
       "      <td>[crash]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>Heard about #earthquake is different cities, s...</td>\n",
       "      <td>[city, heard, earthquake]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>there is a forest fire at spot pond, geese are...</td>\n",
       "      <td>[street, forest, fire]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9</td>\n",
       "      <td>Apocalypse lighting. #Spokane #wildfires</td>\n",
       "      <td>[apocalypse, wildfire]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>11</td>\n",
       "      <td>Typhoon Soudelor kills 28 in China and Taiwan</td>\n",
       "      <td>[china, kill, typhoon]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id                                               text  \\\n",
       "0   0                 Just happened a terrible car crash   \n",
       "1   2  Heard about #earthquake is different cities, s...   \n",
       "2   3  there is a forest fire at spot pond, geese are...   \n",
       "3   9           Apocalypse lighting. #Spokane #wildfires   \n",
       "4  11      Typhoon Soudelor kills 28 in China and Taiwan   \n",
       "\n",
       "                        tags  \n",
       "0                    [crash]  \n",
       "1  [city, heard, earthquake]  \n",
       "2     [street, forest, fire]  \n",
       "3     [apocalypse, wildfire]  \n",
       "4     [china, kill, typhoon]  "
      ]
     },
     "execution_count": 131,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_tags['tags'] = test_tags.new_tokenized.apply(word_finder)\n",
    "test_info = test_tags[['id','text','tags']]\n",
    "\n",
    "test_info.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_info = train_info.copy()\n",
    "train_info[\"cant_tags\"] = train_info[\"tags\"].apply(lambda x: len(x))\n",
    "train_info[\"tags_str\"] = train_info[\"tags\"].apply(lambda x: to_string(x,\",\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>target</th>\n",
       "      <th>text</th>\n",
       "      <th>tags</th>\n",
       "      <th>cant_tags</th>\n",
       "      <th>tags_str</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Our Deeds are the Reason of this #earthquake M...</td>\n",
       "      <td>[reason, earthquake]</td>\n",
       "      <td>2</td>\n",
       "      <td>earthquake,reason</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>Forest fire near La Ronge Sask. Canada</td>\n",
       "      <td>[forest, fire]</td>\n",
       "      <td>2</td>\n",
       "      <td>fire,forest</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  target                                               text  \\\n",
       "0   1       1  Our Deeds are the Reason of this #earthquake M...   \n",
       "1   4       1             Forest fire near La Ronge Sask. Canada   \n",
       "\n",
       "                   tags  cant_tags           tags_str  \n",
       "0  [reason, earthquake]          2  earthquake,reason  \n",
       "1        [forest, fire]          2        fire,forest  "
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_info.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>text</th>\n",
       "      <th>tags</th>\n",
       "      <th>cant_tags</th>\n",
       "      <th>tags_str</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Just happened a terrible car crash</td>\n",
       "      <td>[crash]</td>\n",
       "      <td>1</td>\n",
       "      <td>crash</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>Heard about #earthquake is different cities, s...</td>\n",
       "      <td>[city, heard, earthquake]</td>\n",
       "      <td>3</td>\n",
       "      <td>city,earthquake,heard</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>there is a forest fire at spot pond, geese are...</td>\n",
       "      <td>[street, forest, fire]</td>\n",
       "      <td>3</td>\n",
       "      <td>fire,forest,street</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9</td>\n",
       "      <td>Apocalypse lighting. #Spokane #wildfires</td>\n",
       "      <td>[apocalypse, wildfire]</td>\n",
       "      <td>2</td>\n",
       "      <td>apocalypse,wildfire</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>11</td>\n",
       "      <td>Typhoon Soudelor kills 28 in China and Taiwan</td>\n",
       "      <td>[china, kill, typhoon]</td>\n",
       "      <td>3</td>\n",
       "      <td>china,kill,typhoon</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id                                               text  \\\n",
       "0   0                 Just happened a terrible car crash   \n",
       "1   2  Heard about #earthquake is different cities, s...   \n",
       "2   3  there is a forest fire at spot pond, geese are...   \n",
       "3   9           Apocalypse lighting. #Spokane #wildfires   \n",
       "4  11      Typhoon Soudelor kills 28 in China and Taiwan   \n",
       "\n",
       "                        tags  cant_tags               tags_str  \n",
       "0                    [crash]          1                  crash  \n",
       "1  [city, heard, earthquake]          3  city,earthquake,heard  \n",
       "2     [street, forest, fire]          3     fire,forest,street  \n",
       "3     [apocalypse, wildfire]          2    apocalypse,wildfire  \n",
       "4     [china, kill, typhoon]          3     china,kill,typhoon  "
      ]
     },
     "execution_count": 134,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_tags['tags'] = test_tags.new_tokenized.apply(word_finder)\n",
    "test_info = test_tags[['id','text','tags']]\n",
    "test_info = test_info.copy()\n",
    "test_info[\"cant_tags\"] = test_info[\"tags\"].apply(lambda x: len(x))\n",
    "test_info[\"tags_str\"] = test_info[\"tags\"].apply(lambda x: to_string(x,\",\"))\n",
    "\n",
    "test_info.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  Nuevo feature por tag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [],
   "source": [
    "tags_columns = train_info['tags_str'].str.split(\",\", expand = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_has_tags = pd.get_dummies(tags_columns[0],\"tag\")\n",
    "unique_tag = set(df_has_tags.columns)\n",
    "\n",
    "for i in range(1, len(tags_columns.columns)):\n",
    "    n_links_pd = pd.get_dummies(tags_columns[i],\"tag\")\n",
    "    for domain in unique_tag:\n",
    "        if(domain in set(tags_columns.columns)):\n",
    "            df_has_tags[domain] = df_has_tags[domain] + tags_columns[domain]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "df_has_tags[\"id\"] = train_info[\"id\"]\n",
    "df_has_tags = df_has_tags.set_index(\"id\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tag_abc</th>\n",
       "      <th>tag_ablaze</th>\n",
       "      <th>tag_accident</th>\n",
       "      <th>tag_action</th>\n",
       "      <th>tag_air</th>\n",
       "      <th>tag_aircraft</th>\n",
       "      <th>tag_airplane</th>\n",
       "      <th>tag_airport</th>\n",
       "      <th>tag_ambulance</th>\n",
       "      <th>tag_america</th>\n",
       "      <th>...</th>\n",
       "      <th>tag_wildfire</th>\n",
       "      <th>tag_win</th>\n",
       "      <th>tag_wind</th>\n",
       "      <th>tag_windstorm</th>\n",
       "      <th>tag_woman</th>\n",
       "      <th>tag_wound</th>\n",
       "      <th>tag_wounded</th>\n",
       "      <th>tag_wreck</th>\n",
       "      <th>tag_wrecked</th>\n",
       "      <th>tag_zone</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 367 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    tag_abc  tag_ablaze  tag_accident  tag_action  tag_air  tag_aircraft  \\\n",
       "id                                                                         \n",
       "1         0           0             0           0        0             0   \n",
       "4         0           0             0           0        0             0   \n",
       "5         0           0             0           0        0             0   \n",
       "6         0           0             0           0        0             0   \n",
       "7         0           0             0           0        0             0   \n",
       "\n",
       "    tag_airplane  tag_airport  tag_ambulance  tag_america  ...  tag_wildfire  \\\n",
       "id                                                         ...                 \n",
       "1              0            0              0            0  ...             0   \n",
       "4              0            0              0            0  ...             0   \n",
       "5              0            0              0            0  ...             0   \n",
       "6              0            0              0            0  ...             0   \n",
       "7              0            0              0            0  ...             0   \n",
       "\n",
       "    tag_win  tag_wind  tag_windstorm  tag_woman  tag_wound  tag_wounded  \\\n",
       "id                                                                        \n",
       "1         0         0              0          0          0            0   \n",
       "4         0         0              0          0          0            0   \n",
       "5         0         0              0          0          0            0   \n",
       "6         0         0              0          0          0            0   \n",
       "7         0         0              0          0          0            0   \n",
       "\n",
       "    tag_wreck  tag_wrecked  tag_zone  \n",
       "id                                    \n",
       "1           0            0         0  \n",
       "4           0            0         0  \n",
       "5           0            0         0  \n",
       "6           0            0         0  \n",
       "7           0            0         0  \n",
       "\n",
       "[5 rows x 367 columns]"
      ]
     },
     "execution_count": 138,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_has_tags.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [],
   "source": [
    "tags_columns = test_info['tags_str'].str.split(\",\", expand = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_has_tags = pd.get_dummies(tags_columns[0],\"tag\")\n",
    "unique_tag = set(test_has_tags.columns)\n",
    "\n",
    "for i in range(1, len(tags_columns.columns)):\n",
    "    n_links_pd = pd.get_dummies(tags_columns[i],\"tag\")\n",
    "    for domain in unique_tag:\n",
    "        if(domain in set(tags_columns.columns)):\n",
    "            test_has_tags[domain] = test_has_tags[domain] + tags_columns[domain]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_has_tags[\"id\"] = test_info[\"id\"]\n",
    "test_has_tags = test_has_tags.set_index(\"id\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### BOW: tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [],
   "source": [
    "#bow_tags_index = tags_info['tags_str'].fillna('').index\n",
    "corpus = list(train_info['tags_str'].fillna(''))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [],
   "source": [
    "domain_vectorizer = CountVectorizer(analyzer = \"word\",   \\\n",
    "                             tokenizer = None,    \\\n",
    "                             preprocessor = None, \\\n",
    "                             stop_words = None,   \\\n",
    "                             max_features = 5000)\n",
    "\n",
    "bow_tags = domain_vectorizer.fit_transform(corpus).todense()\n",
    "df_bow_tags = pd.DataFrame(bow_tags)\n",
    "df_bow_tags[\"id\"] = train_info[\"id\"]\n",
    "df_bow_tags = df_bow_tags.set_index(\"id\")\n",
    "df_bow_tags.columns =[\"tag_\"+c for c in domain_vectorizer.get_feature_names()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tag_abc</th>\n",
       "      <th>tag_ablaze</th>\n",
       "      <th>tag_accident</th>\n",
       "      <th>tag_action</th>\n",
       "      <th>tag_air</th>\n",
       "      <th>tag_aircraft</th>\n",
       "      <th>tag_airplane</th>\n",
       "      <th>tag_airport</th>\n",
       "      <th>tag_ambulance</th>\n",
       "      <th>tag_america</th>\n",
       "      <th>...</th>\n",
       "      <th>tag_wind</th>\n",
       "      <th>tag_windstorm</th>\n",
       "      <th>tag_woman</th>\n",
       "      <th>tag_wound</th>\n",
       "      <th>tag_wounded</th>\n",
       "      <th>tag_wreck</th>\n",
       "      <th>tag_wreckage</th>\n",
       "      <th>tag_wrecked</th>\n",
       "      <th>tag_yr</th>\n",
       "      <th>tag_zone</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 400 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    tag_abc  tag_ablaze  tag_accident  tag_action  tag_air  tag_aircraft  \\\n",
       "id                                                                         \n",
       "1         0           0             0           0        0             0   \n",
       "4         0           0             0           0        0             0   \n",
       "5         0           0             0           0        0             0   \n",
       "6         0           0             0           0        0             0   \n",
       "7         0           0             0           0        0             0   \n",
       "\n",
       "    tag_airplane  tag_airport  tag_ambulance  tag_america  ...  tag_wind  \\\n",
       "id                                                         ...             \n",
       "1              0            0              0            0  ...         0   \n",
       "4              0            0              0            0  ...         0   \n",
       "5              0            0              0            0  ...         0   \n",
       "6              0            0              0            0  ...         0   \n",
       "7              0            0              0            0  ...         0   \n",
       "\n",
       "    tag_windstorm  tag_woman  tag_wound  tag_wounded  tag_wreck  tag_wreckage  \\\n",
       "id                                                                              \n",
       "1               0          0          0            0          0             0   \n",
       "4               0          0          0            0          0             0   \n",
       "5               0          0          0            0          0             0   \n",
       "6               0          0          0            0          0             0   \n",
       "7               0          0          0            0          0             0   \n",
       "\n",
       "    tag_wrecked  tag_yr  tag_zone  \n",
       "id                                 \n",
       "1             0       0         0  \n",
       "4             0       0         0  \n",
       "5             0       0         0  \n",
       "6             0       0         0  \n",
       "7             0       0         0  \n",
       "\n",
       "[5 rows x 400 columns]"
      ]
     },
     "execution_count": 144,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_bow_tags.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tf-idf: tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [],
   "source": [
    "#bow_tags_index = tags_info['tags_str'].dropna().index\n",
    "corpus = list(train_info['tags_str'].fillna(''))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tag_abc</th>\n",
       "      <th>tag_ablaze</th>\n",
       "      <th>tag_accident</th>\n",
       "      <th>tag_action</th>\n",
       "      <th>tag_air</th>\n",
       "      <th>tag_aircraft</th>\n",
       "      <th>tag_airplane</th>\n",
       "      <th>tag_airport</th>\n",
       "      <th>tag_ambulance</th>\n",
       "      <th>tag_america</th>\n",
       "      <th>...</th>\n",
       "      <th>tag_wind</th>\n",
       "      <th>tag_windstorm</th>\n",
       "      <th>tag_woman</th>\n",
       "      <th>tag_wound</th>\n",
       "      <th>tag_wounded</th>\n",
       "      <th>tag_wreck</th>\n",
       "      <th>tag_wreckage</th>\n",
       "      <th>tag_wrecked</th>\n",
       "      <th>tag_yr</th>\n",
       "      <th>tag_zone</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 396 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    tag_abc  tag_ablaze  tag_accident  tag_action  tag_air  tag_aircraft  \\\n",
       "id                                                                         \n",
       "1       0.0         0.0           0.0         0.0      0.0           0.0   \n",
       "4       0.0         0.0           0.0         0.0      0.0           0.0   \n",
       "5       0.0         0.0           0.0         0.0      0.0           0.0   \n",
       "6       0.0         0.0           0.0         0.0      0.0           0.0   \n",
       "7       0.0         0.0           0.0         0.0      0.0           0.0   \n",
       "\n",
       "    tag_airplane  tag_airport  tag_ambulance  tag_america  ...  tag_wind  \\\n",
       "id                                                         ...             \n",
       "1            0.0          0.0            0.0          0.0  ...       0.0   \n",
       "4            0.0          0.0            0.0          0.0  ...       0.0   \n",
       "5            0.0          0.0            0.0          0.0  ...       0.0   \n",
       "6            0.0          0.0            0.0          0.0  ...       0.0   \n",
       "7            0.0          0.0            0.0          0.0  ...       0.0   \n",
       "\n",
       "    tag_windstorm  tag_woman  tag_wound  tag_wounded  tag_wreck  tag_wreckage  \\\n",
       "id                                                                              \n",
       "1             0.0        0.0        0.0          0.0        0.0           0.0   \n",
       "4             0.0        0.0        0.0          0.0        0.0           0.0   \n",
       "5             0.0        0.0        0.0          0.0        0.0           0.0   \n",
       "6             0.0        0.0        0.0          0.0        0.0           0.0   \n",
       "7             0.0        0.0        0.0          0.0        0.0           0.0   \n",
       "\n",
       "    tag_wrecked  tag_yr  tag_zone  \n",
       "id                                 \n",
       "1           0.0     0.0       0.0  \n",
       "4           0.0     0.0       0.0  \n",
       "5           0.0     0.0       0.0  \n",
       "6           0.0     0.0       0.0  \n",
       "7           0.0     0.0       0.0  \n",
       "\n",
       "[5 rows x 396 columns]"
      ]
     },
     "execution_count": 146,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Without Smoothing:\n",
    "\n",
    "tf_idf_vec = TfidfVectorizer(use_idf=True, \n",
    "                        smooth_idf=False,  \n",
    "                        ngram_range=(1,1),stop_words='english') # to use only  bigrams ngram_range=(2,2)\n",
    "#transform\n",
    "tf_idf_data = tf_idf_vec.fit_transform(corpus)\n",
    "\n",
    "#create dataframe\n",
    "df_tf_idf_tags=pd.DataFrame(tf_idf_data.toarray(),columns=tf_idf_vec.get_feature_names())\n",
    "df_tf_idf_tags[\"id\"] = train_info[\"id\"]\n",
    "df_tf_idf_tags = df_tf_idf_tags.set_index(\"id\")\n",
    "df_tf_idf_tags.columns =[\"tag_\"+c for c in tf_idf_vec.get_feature_names()]\n",
    "\n",
    "df_tf_idf_tags.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tag_abc</th>\n",
       "      <th>tag_ablaze</th>\n",
       "      <th>tag_accident</th>\n",
       "      <th>tag_action</th>\n",
       "      <th>tag_air</th>\n",
       "      <th>tag_aircraft</th>\n",
       "      <th>tag_airplane</th>\n",
       "      <th>tag_airport</th>\n",
       "      <th>tag_ambulance</th>\n",
       "      <th>tag_america</th>\n",
       "      <th>...</th>\n",
       "      <th>tag_wind</th>\n",
       "      <th>tag_windstorm</th>\n",
       "      <th>tag_woman</th>\n",
       "      <th>tag_wound</th>\n",
       "      <th>tag_wounded</th>\n",
       "      <th>tag_wreck</th>\n",
       "      <th>tag_wreckage</th>\n",
       "      <th>tag_wrecked</th>\n",
       "      <th>tag_yr</th>\n",
       "      <th>tag_zone</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 396 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    tag_abc  tag_ablaze  tag_accident  tag_action  tag_air  tag_aircraft  \\\n",
       "id                                                                         \n",
       "1       0.0         0.0           0.0         0.0      0.0           0.0   \n",
       "4       0.0         0.0           0.0         0.0      0.0           0.0   \n",
       "5       0.0         0.0           0.0         0.0      0.0           0.0   \n",
       "6       0.0         0.0           0.0         0.0      0.0           0.0   \n",
       "7       0.0         0.0           0.0         0.0      0.0           0.0   \n",
       "\n",
       "    tag_airplane  tag_airport  tag_ambulance  tag_america  ...  tag_wind  \\\n",
       "id                                                         ...             \n",
       "1            0.0          0.0            0.0          0.0  ...       0.0   \n",
       "4            0.0          0.0            0.0          0.0  ...       0.0   \n",
       "5            0.0          0.0            0.0          0.0  ...       0.0   \n",
       "6            0.0          0.0            0.0          0.0  ...       0.0   \n",
       "7            0.0          0.0            0.0          0.0  ...       0.0   \n",
       "\n",
       "    tag_windstorm  tag_woman  tag_wound  tag_wounded  tag_wreck  tag_wreckage  \\\n",
       "id                                                                              \n",
       "1             0.0        0.0        0.0          0.0        0.0           0.0   \n",
       "4             0.0        0.0        0.0          0.0        0.0           0.0   \n",
       "5             0.0        0.0        0.0          0.0        0.0           0.0   \n",
       "6             0.0        0.0        0.0          0.0        0.0           0.0   \n",
       "7             0.0        0.0        0.0          0.0        0.0           0.0   \n",
       "\n",
       "    tag_wrecked  tag_yr  tag_zone  \n",
       "id                                 \n",
       "1           0.0     0.0       0.0  \n",
       "4           0.0     0.0       0.0  \n",
       "5           0.0     0.0       0.0  \n",
       "6           0.0     0.0       0.0  \n",
       "7           0.0     0.0       0.0  \n",
       "\n",
       "[5 rows x 396 columns]"
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# With Smoothing:\n",
    "\n",
    "tf_idf_vec_smooth = TfidfVectorizer(use_idf=True, \n",
    "                        smooth_idf=True,  \n",
    "                        ngram_range=(1,1),stop_words='english') # to use only  bigrams ngram_range=(2,2)\n",
    "#transform\n",
    "tf_idf_data = tf_idf_vec_smooth.fit_transform(corpus)\n",
    "\n",
    "#create dataframe\n",
    "df_tf_idf_smooth_tags =pd.DataFrame(tf_idf_data.toarray(),columns=tf_idf_vec_smooth.get_feature_names())\n",
    "df_tf_idf_smooth_tags[\"id\"] = train_info[\"id\"]\n",
    "df_tf_idf_smooth_tags = df_tf_idf_smooth_tags.set_index(\"id\")\n",
    "df_tf_idf_smooth_tags.columns =[\"tag_\"+c for c in tf_idf_vec_smooth.get_feature_names()]\n",
    "\n",
    "df_tf_idf_smooth_tags.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>tag_abc</th>\n",
       "      <th>tag_ablaze</th>\n",
       "      <th>tag_accident</th>\n",
       "      <th>tag_action</th>\n",
       "      <th>tag_air</th>\n",
       "      <th>tag_aircraft</th>\n",
       "      <th>tag_airplane</th>\n",
       "      <th>tag_airport</th>\n",
       "      <th>tag_ambulance</th>\n",
       "      <th>...</th>\n",
       "      <th>tag_wind</th>\n",
       "      <th>tag_windstorm</th>\n",
       "      <th>tag_woman</th>\n",
       "      <th>tag_wound</th>\n",
       "      <th>tag_wounded</th>\n",
       "      <th>tag_wreck</th>\n",
       "      <th>tag_wreckage</th>\n",
       "      <th>tag_wrecked</th>\n",
       "      <th>tag_yr</th>\n",
       "      <th>tag_zone</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 401 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  tag_abc  tag_ablaze  tag_accident  tag_action  tag_air  tag_aircraft  \\\n",
       "0   1        0           0             0           0        0             0   \n",
       "1   4        0           0             0           0        0             0   \n",
       "2   5        0           0             0           0        0             0   \n",
       "3   6        0           0             0           0        0             0   \n",
       "4   7        0           0             0           0        0             0   \n",
       "\n",
       "   tag_airplane  tag_airport  tag_ambulance  ...  tag_wind  tag_windstorm  \\\n",
       "0             0            0              0  ...         0              0   \n",
       "1             0            0              0  ...         0              0   \n",
       "2             0            0              0  ...         0              0   \n",
       "3             0            0              0  ...         0              0   \n",
       "4             0            0              0  ...         0              0   \n",
       "\n",
       "   tag_woman  tag_wound  tag_wounded  tag_wreck  tag_wreckage  tag_wrecked  \\\n",
       "0          0          0            0          0             0            0   \n",
       "1          0          0            0          0             0            0   \n",
       "2          0          0            0          0             0            0   \n",
       "3          0          0            0          0             0            0   \n",
       "4          0          0            0          0             0            0   \n",
       "\n",
       "   tag_yr  tag_zone  \n",
       "0       0         0  \n",
       "1       0         0  \n",
       "2       0         0  \n",
       "3       0         0  \n",
       "4       0         0  \n",
       "\n",
       "[5 rows x 401 columns]"
      ]
     },
     "execution_count": 148,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_bow_tags=df_bow_tags.reset_index()\n",
    "df_bow_tags.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>tag_abc</th>\n",
       "      <th>tag_ablaze</th>\n",
       "      <th>tag_accident</th>\n",
       "      <th>tag_action</th>\n",
       "      <th>tag_air</th>\n",
       "      <th>tag_aircraft</th>\n",
       "      <th>tag_airplane</th>\n",
       "      <th>tag_airport</th>\n",
       "      <th>tag_ambulance</th>\n",
       "      <th>...</th>\n",
       "      <th>tag_wind</th>\n",
       "      <th>tag_windstorm</th>\n",
       "      <th>tag_woman</th>\n",
       "      <th>tag_wound</th>\n",
       "      <th>tag_wounded</th>\n",
       "      <th>tag_wreck</th>\n",
       "      <th>tag_wreckage</th>\n",
       "      <th>tag_wrecked</th>\n",
       "      <th>tag_yr</th>\n",
       "      <th>tag_zone</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 397 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  tag_abc  tag_ablaze  tag_accident  tag_action  tag_air  tag_aircraft  \\\n",
       "0   1      0.0         0.0           0.0         0.0      0.0           0.0   \n",
       "1   4      0.0         0.0           0.0         0.0      0.0           0.0   \n",
       "2   5      0.0         0.0           0.0         0.0      0.0           0.0   \n",
       "3   6      0.0         0.0           0.0         0.0      0.0           0.0   \n",
       "4   7      0.0         0.0           0.0         0.0      0.0           0.0   \n",
       "\n",
       "   tag_airplane  tag_airport  tag_ambulance  ...  tag_wind  tag_windstorm  \\\n",
       "0           0.0          0.0            0.0  ...       0.0            0.0   \n",
       "1           0.0          0.0            0.0  ...       0.0            0.0   \n",
       "2           0.0          0.0            0.0  ...       0.0            0.0   \n",
       "3           0.0          0.0            0.0  ...       0.0            0.0   \n",
       "4           0.0          0.0            0.0  ...       0.0            0.0   \n",
       "\n",
       "   tag_woman  tag_wound  tag_wounded  tag_wreck  tag_wreckage  tag_wrecked  \\\n",
       "0        0.0        0.0          0.0        0.0           0.0          0.0   \n",
       "1        0.0        0.0          0.0        0.0           0.0          0.0   \n",
       "2        0.0        0.0          0.0        0.0           0.0          0.0   \n",
       "3        0.0        0.0          0.0        0.0           0.0          0.0   \n",
       "4        0.0        0.0          0.0        0.0           0.0          0.0   \n",
       "\n",
       "   tag_yr  tag_zone  \n",
       "0     0.0       0.0  \n",
       "1     0.0       0.0  \n",
       "2     0.0       0.0  \n",
       "3     0.0       0.0  \n",
       "4     0.0       0.0  \n",
       "\n",
       "[5 rows x 397 columns]"
      ]
     },
     "execution_count": 149,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_tf_idf_smooth_tags = df_tf_idf_smooth_tags.reset_index()\n",
    "df_tf_idf_smooth_tags.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"df_tf_idf_smooth_tags.to_csv('features/train/features_tags_tf_idf.csv', encoding='utf-8')\\ndf_bow_tags.to_csv('features/train/features_tags_bow.csv', encoding='utf-8')\""
      ]
     },
     "execution_count": 150,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# DOWNLOAD CSV LINKS\n",
    "\n",
    "# TRAIN\n",
    "\n",
    "'''df_tf_idf_smooth_tags.to_csv('features/train/features_tags_tf_idf.csv', encoding='utf-8')\n",
    "df_bow_tags.to_csv('features/train/features_tags_bow.csv', encoding='utf-8')'''\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus = list(test_info['tags_str'].fillna(''))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tag_abc</th>\n",
       "      <th>tag_ablaze</th>\n",
       "      <th>tag_accident</th>\n",
       "      <th>tag_action</th>\n",
       "      <th>tag_air</th>\n",
       "      <th>tag_aircraft</th>\n",
       "      <th>tag_airplane</th>\n",
       "      <th>tag_airport</th>\n",
       "      <th>tag_ambulance</th>\n",
       "      <th>tag_america</th>\n",
       "      <th>...</th>\n",
       "      <th>tag_wind</th>\n",
       "      <th>tag_windstorm</th>\n",
       "      <th>tag_woman</th>\n",
       "      <th>tag_wound</th>\n",
       "      <th>tag_wounded</th>\n",
       "      <th>tag_wreck</th>\n",
       "      <th>tag_wreckage</th>\n",
       "      <th>tag_wrecked</th>\n",
       "      <th>tag_yr</th>\n",
       "      <th>tag_zone</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 396 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    tag_abc  tag_ablaze  tag_accident  tag_action  tag_air  tag_aircraft  \\\n",
       "id                                                                         \n",
       "0       0.0         0.0           0.0         0.0      0.0           0.0   \n",
       "2       0.0         0.0           0.0         0.0      0.0           0.0   \n",
       "3       0.0         0.0           0.0         0.0      0.0           0.0   \n",
       "9       0.0         0.0           0.0         0.0      0.0           0.0   \n",
       "11      0.0         0.0           0.0         0.0      0.0           0.0   \n",
       "\n",
       "    tag_airplane  tag_airport  tag_ambulance  tag_america  ...  tag_wind  \\\n",
       "id                                                         ...             \n",
       "0            0.0          0.0            0.0          0.0  ...       0.0   \n",
       "2            0.0          0.0            0.0          0.0  ...       0.0   \n",
       "3            0.0          0.0            0.0          0.0  ...       0.0   \n",
       "9            0.0          0.0            0.0          0.0  ...       0.0   \n",
       "11           0.0          0.0            0.0          0.0  ...       0.0   \n",
       "\n",
       "    tag_windstorm  tag_woman  tag_wound  tag_wounded  tag_wreck  tag_wreckage  \\\n",
       "id                                                                              \n",
       "0             0.0        0.0        0.0          0.0        0.0           0.0   \n",
       "2             0.0        0.0        0.0          0.0        0.0           0.0   \n",
       "3             0.0        0.0        0.0          0.0        0.0           0.0   \n",
       "9             0.0        0.0        0.0          0.0        0.0           0.0   \n",
       "11            0.0        0.0        0.0          0.0        0.0           0.0   \n",
       "\n",
       "    tag_wrecked  tag_yr  tag_zone  \n",
       "id                                 \n",
       "0           0.0     0.0       0.0  \n",
       "2           0.0     0.0       0.0  \n",
       "3           0.0     0.0       0.0  \n",
       "9           0.0     0.0       0.0  \n",
       "11          0.0     0.0       0.0  \n",
       "\n",
       "[5 rows x 396 columns]"
      ]
     },
     "execution_count": 152,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#transform\n",
    "tf_idf_data = tf_idf_vec.transform(corpus)\n",
    "\n",
    "#create dataframe\n",
    "test_tf_idf_tags=pd.DataFrame(tf_idf_data.toarray(),columns=tf_idf_vec.get_feature_names())\n",
    "test_tf_idf_tags[\"id\"] = test_info[\"id\"]\n",
    "test_tf_idf_tags = test_tf_idf_tags.set_index(\"id\")\n",
    "test_tf_idf_tags.columns =[\"tag_\"+c for c in tf_idf_vec.get_feature_names()]\n",
    "\n",
    "test_tf_idf_tags.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tag_abc</th>\n",
       "      <th>tag_ablaze</th>\n",
       "      <th>tag_accident</th>\n",
       "      <th>tag_action</th>\n",
       "      <th>tag_air</th>\n",
       "      <th>tag_aircraft</th>\n",
       "      <th>tag_airplane</th>\n",
       "      <th>tag_airport</th>\n",
       "      <th>tag_ambulance</th>\n",
       "      <th>tag_america</th>\n",
       "      <th>...</th>\n",
       "      <th>tag_wind</th>\n",
       "      <th>tag_windstorm</th>\n",
       "      <th>tag_woman</th>\n",
       "      <th>tag_wound</th>\n",
       "      <th>tag_wounded</th>\n",
       "      <th>tag_wreck</th>\n",
       "      <th>tag_wreckage</th>\n",
       "      <th>tag_wrecked</th>\n",
       "      <th>tag_yr</th>\n",
       "      <th>tag_zone</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 396 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    tag_abc  tag_ablaze  tag_accident  tag_action  tag_air  tag_aircraft  \\\n",
       "id                                                                         \n",
       "0       0.0         0.0           0.0         0.0      0.0           0.0   \n",
       "2       0.0         0.0           0.0         0.0      0.0           0.0   \n",
       "3       0.0         0.0           0.0         0.0      0.0           0.0   \n",
       "9       0.0         0.0           0.0         0.0      0.0           0.0   \n",
       "11      0.0         0.0           0.0         0.0      0.0           0.0   \n",
       "\n",
       "    tag_airplane  tag_airport  tag_ambulance  tag_america  ...  tag_wind  \\\n",
       "id                                                         ...             \n",
       "0            0.0          0.0            0.0          0.0  ...       0.0   \n",
       "2            0.0          0.0            0.0          0.0  ...       0.0   \n",
       "3            0.0          0.0            0.0          0.0  ...       0.0   \n",
       "9            0.0          0.0            0.0          0.0  ...       0.0   \n",
       "11           0.0          0.0            0.0          0.0  ...       0.0   \n",
       "\n",
       "    tag_windstorm  tag_woman  tag_wound  tag_wounded  tag_wreck  tag_wreckage  \\\n",
       "id                                                                              \n",
       "0             0.0        0.0        0.0          0.0        0.0           0.0   \n",
       "2             0.0        0.0        0.0          0.0        0.0           0.0   \n",
       "3             0.0        0.0        0.0          0.0        0.0           0.0   \n",
       "9             0.0        0.0        0.0          0.0        0.0           0.0   \n",
       "11            0.0        0.0        0.0          0.0        0.0           0.0   \n",
       "\n",
       "    tag_wrecked  tag_yr  tag_zone  \n",
       "id                                 \n",
       "0           0.0     0.0       0.0  \n",
       "2           0.0     0.0       0.0  \n",
       "3           0.0     0.0       0.0  \n",
       "9           0.0     0.0       0.0  \n",
       "11          0.0     0.0       0.0  \n",
       "\n",
       "[5 rows x 396 columns]"
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#transform\n",
    "tf_idf_data = tf_idf_vec_smooth.transform(corpus)\n",
    "\n",
    "#create dataframe\n",
    "test_tf_idf_smooth_tags =pd.DataFrame(tf_idf_data.toarray(),columns=tf_idf_vec_smooth.get_feature_names())\n",
    "test_tf_idf_smooth_tags[\"id\"] = test_info[\"id\"]\n",
    "test_tf_idf_smooth_tags = test_tf_idf_smooth_tags.set_index(\"id\")\n",
    "test_tf_idf_smooth_tags.columns =[\"tag_\"+c for c in tf_idf_vec_smooth.get_feature_names()]\n",
    "\n",
    "test_tf_idf_smooth_tags.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [],
   "source": [
    "bow_tags = domain_vectorizer.transform(corpus).todense()\n",
    "test_bow_tags = pd.DataFrame(bow_tags)\n",
    "test_bow_tags[\"id\"] = test_info[\"id\"]\n",
    "test_bow_tags = test_bow_tags.set_index(\"id\")\n",
    "test_bow_tags.columns =[\"tag_\"+c for c in domain_vectorizer.get_feature_names()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_bow_tags=test_bow_tags.reset_index()\n",
    "test_tf_idf_smooth_tags = test_tf_idf_smooth_tags.reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [],
   "source": [
    "#test_tf_idf_smooth_tags.to_csv('features/test/features_tags_tf_idf.csv', encoding='utf-8')\n",
    "#test_bow_tags.to_csv('features/test/features_tags_bow.csv', encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Variables categóricas"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Keywords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train\n",
    "\n",
    "keywords_categorical_features = train_keywords.copy() \n",
    "keywords_categorical_features = keywords_agregar_features_categoricos(keywords_categorical_features,True).drop(columns=['keyword','location','text'])\n",
    "\n",
    "keywords_numerical_features = train_keywords.copy()\n",
    "keywords_numerical_features = keywords_agregar_features_numericos(keywords_numerical_features, True).drop(columns=['location','text','keyword'])\n",
    "del keywords_numerical_features[\"target\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 230,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>target</th>\n",
       "      <th>keywords_in_text</th>\n",
       "      <th>text_contains_keyword</th>\n",
       "      <th>has_keyword</th>\n",
       "      <th>keywords_quantity</th>\n",
       "      <th>keywords_mean</th>\n",
       "      <th>keyword_is_hashtag</th>\n",
       "      <th>keyword_frequency</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>[earthquake]</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.004525</td>\n",
       "      <td>1</td>\n",
       "      <td>0.227027</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>[fire, forest fire]</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.004525</td>\n",
       "      <td>0</td>\n",
       "      <td>0.237838</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>[evacuation]</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.004525</td>\n",
       "      <td>0</td>\n",
       "      <td>0.210811</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>[evacuation, fire, wildfire]</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.004525</td>\n",
       "      <td>0</td>\n",
       "      <td>0.210811</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>[fire, smoke, wildfire]</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.004525</td>\n",
       "      <td>0</td>\n",
       "      <td>0.237838</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  target              keywords_in_text  text_contains_keyword  \\\n",
       "0   1       1                  [earthquake]                      1   \n",
       "1   4       1           [fire, forest fire]                      1   \n",
       "2   5       1                  [evacuation]                      1   \n",
       "3   6       1  [evacuation, fire, wildfire]                      1   \n",
       "4   7       1       [fire, smoke, wildfire]                      1   \n",
       "\n",
       "   has_keyword  keywords_quantity  keywords_mean  keyword_is_hashtag  \\\n",
       "0            1                  1       0.004525                   1   \n",
       "1            1                  1       0.004525                   0   \n",
       "2            1                  1       0.004525                   0   \n",
       "3            1                  1       0.004525                   0   \n",
       "4            1                  1       0.004525                   0   \n",
       "\n",
       "   keyword_frequency  \n",
       "0           0.227027  \n",
       "1           0.237838  \n",
       "2           0.210811  \n",
       "3           0.210811  \n",
       "4           0.237838  "
      ]
     },
     "execution_count": 230,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "keywords_categorical_train = keywords_categorical_features.merge(keywords_numerical_features, on=\"id\", how=\"left\")\n",
    "keywords_categorical_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "metadata": {},
   "outputs": [],
   "source": [
    "# test\n",
    "\n",
    "keywords_categorical_features = test_keywords.copy() \n",
    "keywords_categorical_features = keywords_agregar_features_categoricos(keywords_categorical_features,True).drop(columns=['keyword','location','text'])\n",
    "\n",
    "keywords_numerical_features = test_keywords.copy()\n",
    "keywords_numerical_features = keywords_agregar_features_numericos(keywords_numerical_features, True).drop(columns=['location','text','keyword'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 234,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>keywords_in_text</th>\n",
       "      <th>text_contains_keyword</th>\n",
       "      <th>has_keyword</th>\n",
       "      <th>keywords_quantity</th>\n",
       "      <th>keywords_mean</th>\n",
       "      <th>keyword_is_hashtag</th>\n",
       "      <th>keyword_frequency</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>[crash]</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.004525</td>\n",
       "      <td>0</td>\n",
       "      <td>0.227027</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>[earthquake]</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.004525</td>\n",
       "      <td>1</td>\n",
       "      <td>0.237838</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>[fire, forest fire]</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.004525</td>\n",
       "      <td>0</td>\n",
       "      <td>0.210811</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9</td>\n",
       "      <td>[apocalypse, fire, wildfire]</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.004525</td>\n",
       "      <td>0</td>\n",
       "      <td>0.210811</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>11</td>\n",
       "      <td>[typhoon]</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.004525</td>\n",
       "      <td>0</td>\n",
       "      <td>0.237838</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id              keywords_in_text  text_contains_keyword  has_keyword  \\\n",
       "0   0                       [crash]                      1            1   \n",
       "1   2                  [earthquake]                      1            1   \n",
       "2   3           [fire, forest fire]                      1            1   \n",
       "3   9  [apocalypse, fire, wildfire]                      1            1   \n",
       "4  11                     [typhoon]                      1            1   \n",
       "\n",
       "   keywords_quantity  keywords_mean  keyword_is_hashtag  keyword_frequency  \n",
       "0                  1       0.004525                   0           0.227027  \n",
       "1                  1       0.004525                   1           0.237838  \n",
       "2                  1       0.004525                   0           0.210811  \n",
       "3                  1       0.004525                   0           0.210811  \n",
       "4                  1       0.004525                   0           0.237838  "
      ]
     },
     "execution_count": 234,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "keywords_categorical_test = keywords_categorical_features.merge(keywords_numerical_features, on=\"id\", how=\"left\")\n",
    "keywords_categorical_test.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 282,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>target</th>\n",
       "      <th>#palabras</th>\n",
       "      <th>#palabras_unicas</th>\n",
       "      <th>#caracteres</th>\n",
       "      <th>#stopwords</th>\n",
       "      <th>#puntuacion</th>\n",
       "      <th>#capitalize</th>\n",
       "      <th>#mayusculas</th>\n",
       "      <th>#silabas</th>\n",
       "      <th>...</th>\n",
       "      <th>#mayusculas_binned</th>\n",
       "      <th>#silabas_binned</th>\n",
       "      <th>#caracteres_especiales_binned</th>\n",
       "      <th>#tagged_users</th>\n",
       "      <th>#pronombres</th>\n",
       "      <th>tagged_users</th>\n",
       "      <th>hashtags</th>\n",
       "      <th>#hashtags</th>\n",
       "      <th>has_hashtag</th>\n",
       "      <th>pos_tag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>13</td>\n",
       "      <td>71</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>19</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td>earthquake</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>PRP$ NNS VBP DT NNP IN DT # NN NNP NNP NNP PRP...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>40</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NNP NN IN NNP NNP NNP . NNP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>22</td>\n",
       "      <td>20</td>\n",
       "      <td>135</td>\n",
       "      <td>9</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>27</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>DT NNS VBD TO VB IN NN '' VBP VBG VBN IN NNS ....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>67</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>17</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td>wildfires</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>CD NNS JJ # NNS VBP NNS IN NNP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>16</td>\n",
       "      <td>15</td>\n",
       "      <td>90</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>24</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td>Alaska wildfires</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>RB VBN VBD DT NN IN NNP # NNP IN NN IN # NNS V...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 28 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  target  #palabras  #palabras_unicas  #caracteres  #stopwords  \\\n",
       "0   1       1         13                13           71           5   \n",
       "1   4       1          7                 7           40           0   \n",
       "2   5       1         22                20          135           9   \n",
       "3   6       1          8                 8           67           1   \n",
       "4   7       1         16                15           90           6   \n",
       "\n",
       "   #puntuacion  #capitalize  #mayusculas  #silabas  ...  #mayusculas_binned  \\\n",
       "0            1            5            1        19  ...                   0   \n",
       "1            1            5            0        10  ...                   0   \n",
       "2            3            2            0        27  ...                   0   \n",
       "3            2            1            0        17  ...                   0   \n",
       "4            2            3            0        24  ...                   0   \n",
       "\n",
       "   #silabas_binned  #caracteres_especiales_binned  #tagged_users  #pronombres  \\\n",
       "0                1                              0              0            0   \n",
       "1                0                              0              0            0   \n",
       "2                3                              1              0            0   \n",
       "3                1                              0              0            0   \n",
       "4                2                              0              0            0   \n",
       "\n",
       "   tagged_users          hashtags  #hashtags  has_hashtag  \\\n",
       "0                      earthquake          1            1   \n",
       "1                                          0            0   \n",
       "2                                          0            0   \n",
       "3                       wildfires          1            1   \n",
       "4                Alaska wildfires          2            1   \n",
       "\n",
       "                                             pos_tag  \n",
       "0  PRP$ NNS VBP DT NNP IN DT # NN NNP NNP NNP PRP...  \n",
       "1                       NNP NN IN NNP NNP NNP . NNP   \n",
       "2  DT NNS VBD TO VB IN NN '' VBP VBG VBN IN NNS ....  \n",
       "3                    CD NNS JJ # NNS VBP NNS IN NNP   \n",
       "4  RB VBN VBD DT NN IN NNP # NNP IN NN IN # NNS V...  \n",
       "\n",
       "[5 rows x 28 columns]"
      ]
     },
     "execution_count": 282,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text_categorical_features_train = text_numerical_features\n",
    "text_categorical_features_train[\"#pronombres\"] = df_pronombres[\"#pronombres\"]\n",
    "text_categorical_features_train[\"#tagged_users\"] = tagged_users_numerical_features[\"#tagged_users\"]\n",
    "text_categorical_features_train[\"tagged_users\"] = tagged_users_categorical_features[\"tagged_users\"]\n",
    "text_categorical_features_train[\"hashtags\"] = hashtags_categorical_features[\"hashtags\"]\n",
    "text_categorical_features_train[\"#hashtags\"] = hashtags_numerical_features[\"#hashtags\"]\n",
    "text_categorical_features_train[\"has_hashtag\"] = hashtags_numerical_features[\"has_hashtag\"]\n",
    "text_categorical_features_train[\"pos_tag\"] = df_pos_tag[\"pos_tag\"]\n",
    "\n",
    "text_categorical_features_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 291,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>#palabras</th>\n",
       "      <th>#palabras_unicas</th>\n",
       "      <th>#caracteres</th>\n",
       "      <th>#stopwords</th>\n",
       "      <th>#puntuacion</th>\n",
       "      <th>#capitalize</th>\n",
       "      <th>#mayusculas</th>\n",
       "      <th>#silabas</th>\n",
       "      <th>promedio_len_word</th>\n",
       "      <th>...</th>\n",
       "      <th>#mayusculas_binned</th>\n",
       "      <th>#silabas_binned</th>\n",
       "      <th>#caracteres_especiales_binned</th>\n",
       "      <th>#pronombres</th>\n",
       "      <th>#tagged_users</th>\n",
       "      <th>tagged_users</th>\n",
       "      <th>hashtags</th>\n",
       "      <th>#hashtags</th>\n",
       "      <th>has_hashtag</th>\n",
       "      <th>pos_tag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>36</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>4.833333</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>RB VBD DT JJ NN NN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>66</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>17</td>\n",
       "      <td>6.222222</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td>earthquake</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>NNP IN # NN VBZ JJ NNS , VB JJ NN .</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>19</td>\n",
       "      <td>19</td>\n",
       "      <td>98</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>17</td>\n",
       "      <td>4.105263</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>EX VBZ DT JJ NN IN NN NN , JJ VBP VBG IN DT NN...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>42</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>9.250000</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td>Spokane wildfires</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>NNP VBG . # NNP # NNS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>11</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>47</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>15</td>\n",
       "      <td>4.750000</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NNP NNP VBZ CD IN NNP CC NNP</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 27 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  #palabras  #palabras_unicas  #caracteres  #stopwords  #puntuacion  \\\n",
       "0   0          6                 6           36           1            0   \n",
       "1   2          9                 9           66           2            3   \n",
       "2   3         19                19           98           8            2   \n",
       "3   9          4                 4           42           0            3   \n",
       "4  11          8                 8           47           2            0   \n",
       "\n",
       "   #capitalize  #mayusculas  #silabas  promedio_len_word  ...  \\\n",
       "0            1            0         7           4.833333  ...   \n",
       "1            1            0        17           6.222222  ...   \n",
       "2            1            1        17           4.105263  ...   \n",
       "3            2            0        11           9.250000  ...   \n",
       "4            4            0        15           4.750000  ...   \n",
       "\n",
       "   #mayusculas_binned  #silabas_binned  #caracteres_especiales_binned  \\\n",
       "0                   0                0                              0   \n",
       "1                   0                1                              1   \n",
       "2                   0                1                              0   \n",
       "3                   0                0                              1   \n",
       "4                   0                1                              0   \n",
       "\n",
       "   #pronombres  #tagged_users  tagged_users           hashtags  #hashtags  \\\n",
       "0            0              0                                           0   \n",
       "1            0              0                       earthquake          1   \n",
       "2            1              0                                           0   \n",
       "3            0              0                Spokane wildfires          2   \n",
       "4            0              0                                           0   \n",
       "\n",
       "   has_hashtag                                            pos_tag  \n",
       "0            0                                RB VBD DT JJ NN NN   \n",
       "1            1               NNP IN # NN VBZ JJ NNS , VB JJ NN .   \n",
       "2            0  EX VBZ DT JJ NN IN NN NN , JJ VBP VBG IN DT NN...  \n",
       "3            1                             NNP VBG . # NNP # NNS   \n",
       "4            0                      NNP NNP VBZ CD IN NNP CC NNP   \n",
       "\n",
       "[5 rows x 27 columns]"
      ]
     },
     "execution_count": 291,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text_categorical_features_test = test_text_numerical_features\n",
    "text_categorical_features_test[\"#pronombres\"] = df_pronombres_test[\"#pronombres\"]\n",
    "text_categorical_features_test[\"#tagged_users\"] = test_tagged_users_numerical_features[\"#tagged_users\"]\n",
    "text_categorical_features_test[\"tagged_users\"] = test_tagged_users_categorical_features[\"tagged_users\"]\n",
    "text_categorical_features_test[\"hashtags\"] = test_hashtags_categorical_features[\"hashtags\"]\n",
    "text_categorical_features_test[\"#hashtags\"] = test_hashtags_numerical_features[\"#hashtags\"]\n",
    "text_categorical_features_test[\"has_hashtag\"] = test_hashtags_numerical_features[\"has_hashtag\"]\n",
    "text_categorical_features_test[\"pos_tag\"] = df_pos_tag_test[\"pos_tag\"]\n",
    "\n",
    "text_categorical_features_test.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Location"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>cities</th>\n",
       "      <th>counties</th>\n",
       "      <th>country</th>\n",
       "      <th>states</th>\n",
       "      <th>continent</th>\n",
       "      <th>has_location</th>\n",
       "      <th>has_country</th>\n",
       "      <th>has_city</th>\n",
       "      <th>has_county</th>\n",
       "      <th>has_state</th>\n",
       "      <th>has_continent</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id cities counties country states continent  has_location  has_country  \\\n",
       "0   1   None     None    None   None      None             0            0   \n",
       "1   4   None     None    None   None      None             0            0   \n",
       "2   5   None     None    None   None      None             0            0   \n",
       "3   6   None     None    None   None      None             0            0   \n",
       "4   7   None     None    None   None      None             0            0   \n",
       "\n",
       "   has_city  has_county  has_state  has_continent  \n",
       "0         0           0          0              0  \n",
       "1         0           0          0              0  \n",
       "2         0           0          0              0  \n",
       "3         0           0          0              0  \n",
       "4         0           0          0              0  "
      ]
     },
     "execution_count": 189,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# train\n",
    "\n",
    "location_categorical_train = train_location[[\"id\",\"cities_str\",\"counties_str\",\"country_str\",\"states_str\", \"continent_str\",\"has_location\", \"has_country\", \"has_city\",\"has_county\", \"has_state\",\"has_continent\"]]\n",
    "location_categorical_train.columns = [\"id\",\"cities\",\"counties\",\"country\",\"states\", \"continent\",\"has_location\", \"has_country\", \"has_city\",\"has_county\", \"has_state\",\"has_continent\"]\n",
    "location_categorical_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>cities</th>\n",
       "      <th>counties</th>\n",
       "      <th>country</th>\n",
       "      <th>states</th>\n",
       "      <th>continent</th>\n",
       "      <th>has_location</th>\n",
       "      <th>has_country</th>\n",
       "      <th>has_city</th>\n",
       "      <th>has_county</th>\n",
       "      <th>has_state</th>\n",
       "      <th>has_continent</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>11</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id cities counties country states continent  has_location  has_country  \\\n",
       "0   0   None     None    None   None      None             0            0   \n",
       "1   2   None     None    None   None      None             0            0   \n",
       "2   3   None     None    None   None      None             0            0   \n",
       "3   9   None     None    None   None      None             0            0   \n",
       "4  11   None     None    None   None      None             0            0   \n",
       "\n",
       "   has_city  has_county  has_state  has_continent  \n",
       "0         0           0          0              0  \n",
       "1         0           0          0              0  \n",
       "2         0           0          0              0  \n",
       "3         0           0          0              0  \n",
       "4         0           0          0              0  "
      ]
     },
     "execution_count": 190,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# test\n",
    "\n",
    "location_categorical_test = test_location[[\"id\",\"cities_str\",\"counties_str\",\"country_str\",\"states_str\", \"continent_str\",\"has_location\", \"has_country\", \"has_city\",\"has_county\", \"has_state\",\"has_continent\"]]\n",
    "location_categorical_test.columns = [\"id\",\"cities\",\"counties\",\"country\",\"states\", \"continent\",\"has_location\", \"has_country\", \"has_city\",\"has_county\", \"has_state\",\"has_continent\"]\n",
    "location_categorical_test.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train\n",
    "\n",
    "links_categorical_train = train[[\"id\"]].copy()\n",
    "links_categorical_train = links_categorical_train.merge(links_train.reset_index(), how=\"left\", on=\"id\")\n",
    "links_categorical_train[[\"links_cant\",\"cant_failed_links\"]] = links_categorical_train[[\"links_cant\",\"cant_failed_links\"]].fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>links_cant</th>\n",
       "      <th>links_encription_type</th>\n",
       "      <th>cant_failed_links</th>\n",
       "      <th>links_domain_str</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  links_cant links_encription_type  cant_failed_links links_domain_str\n",
       "0   1         0.0                   NaN                0.0              NaN\n",
       "1   4         0.0                   NaN                0.0              NaN\n",
       "2   5         0.0                   NaN                0.0              NaN\n",
       "3   6         0.0                   NaN                0.0              NaN\n",
       "4   7         0.0                   NaN                0.0              NaN"
      ]
     },
     "execution_count": 214,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "links_categorical_train = links_categorical_train[[\"id\",\"links_cant\", \"links_encription_type\", \"cant_failed_links\",\"links_domain_str\"]]\n",
    "links_categorical_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "metadata": {},
   "outputs": [],
   "source": [
    "# test\n",
    "\n",
    "links_categorical_test = test[[\"id\"]].copy()\n",
    "links_categorical_test = links_categorical_test.merge(links_test.reset_index(), how=\"left\", on=\"id\")\n",
    "links_categorical_test[[\"links_cant\",\"cant_failed_links\"]] = links_categorical_test[[\"links_cant\",\"cant_failed_links\"]].fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>links_cant</th>\n",
       "      <th>links_encription_type</th>\n",
       "      <th>cant_failed_links</th>\n",
       "      <th>links_domain_str</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>11</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  links_cant links_encription_type  cant_failed_links links_domain_str\n",
       "0   0         0.0                   NaN                0.0              NaN\n",
       "1   2         0.0                   NaN                0.0              NaN\n",
       "2   3         0.0                   NaN                0.0              NaN\n",
       "3   9         0.0                   NaN                0.0              NaN\n",
       "4  11         0.0                   NaN                0.0              NaN"
      ]
     },
     "execution_count": 216,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "links_categorical_test = links_categorical_test[[\"id\",\"links_cant\", \"links_encription_type\", \"cant_failed_links\",\"links_domain_str\"]]\n",
    "links_categorical_test.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 297,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>cant_tags</th>\n",
       "      <th>tags_str</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>earthquake,reason</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>fire,forest</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>evacuation,officer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>4</td>\n",
       "      <td>california,evacuation,people,wildfire</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7</td>\n",
       "      <td>3</td>\n",
       "      <td>photo,smoke,wildfire</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  cant_tags                               tags_str\n",
       "0   1          2                      earthquake,reason\n",
       "1   4          2                            fire,forest\n",
       "2   5          2                     evacuation,officer\n",
       "3   6          4  california,evacuation,people,wildfire\n",
       "4   7          3                   photo,smoke,wildfire"
      ]
     },
     "execution_count": 297,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tags_categorical_train = train_info[[\"id\",\"cant_tags\",\"tags_str\"]]\n",
    "tags_categorical_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 298,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>cant_tags</th>\n",
       "      <th>tags_str</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>crash</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>city,earthquake,heard</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>fire,forest,street</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>apocalypse,wildfire</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>11</td>\n",
       "      <td>3</td>\n",
       "      <td>china,kill,typhoon</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  cant_tags               tags_str\n",
       "0   0          1                  crash\n",
       "1   2          3  city,earthquake,heard\n",
       "2   3          3     fire,forest,street\n",
       "3   9          2    apocalypse,wildfire\n",
       "4  11          3     china,kill,typhoon"
      ]
     },
     "execution_count": 298,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tags_categorical_test =  test_info[[\"id\",\"cant_tags\",\"tags_str\"]]\n",
    "tags_categorical_test.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save csv categorical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 299,
   "metadata": {},
   "outputs": [],
   "source": [
    "keywords_categorical_train.to_csv('features/train/keywords_categorical_train.csv', encoding='utf-8')\n",
    "keywords_categorical_test.to_csv('features/test/keywords_categorical_test.csv', encoding='utf-8')\n",
    "\n",
    "text_categorical_features_train.to_csv('features/train/text_categorical_train.csv', encoding='utf-8')\n",
    "text_categorical_features_test.to_csv('features/test/text_categorical_test.csv', encoding='utf-8')\n",
    "\n",
    "location_categorical_train.to_csv('features/train/location_categorical_train.csv', encoding='utf-8')\n",
    "location_categorical_test.to_csv('features/test/location_categorical_test.csv', encoding='utf-8')\n",
    "\n",
    "links_categorical_train.to_csv('features/train/links_categorical_train.csv', encoding='utf-8')\n",
    "links_categorical_test.to_csv('features/test/links_categorical_test.csv', encoding='utf-8')\n",
    "\n",
    "tags_categorical_train.to_csv('features/train/tags_categorical_train.csv', encoding='utf-8')\n",
    "tags_categorical_test.to_csv('features/test/tags_categorical_test.csv', encoding='utf-8')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}