tokenization.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NLTK"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Install NLTK"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 381,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: nltk in /home/ana/.local/lib/python3.5/site-packages\n",
      "Requirement already satisfied: six in /home/ana/.local/lib/python3.5/site-packages (from nltk)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You are using pip version 9.0.1, however version 21.0.1 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "pip install nltk"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Download models or corpora"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 510,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "showing info https://raw.githubusercontent.com/nltk/nltk_data/gh-pages/index.xml\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "python -m nltk.downloader"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Import and use"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /home/ana/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import nltk\n",
    "nltk.download('punkt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 511,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweet = \"RT @lOR42wsOEFcv3f: I fall too fast, crash too hard, forgive too easily and care too much... :( #amiright\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 518,
   "metadata": {},
   "outputs": [],
   "source": [
    "query = 'fast'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 519,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "31"
      ]
     },
     "execution_count": 519,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweet.find(query)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Tokenization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 513,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['RT',\n",
       " '@lOR42wsOEFcv3f:',\n",
       " 'I',\n",
       " 'fall',\n",
       " 'too',\n",
       " 'fast,',\n",
       " 'crash',\n",
       " 'too',\n",
       " 'hard,',\n",
       " 'forgive',\n",
       " 'too',\n",
       " 'easily',\n",
       " 'and',\n",
       " 'care',\n",
       " 'too',\n",
       " 'much...',\n",
       " ':(',\n",
       " '#amiright']"
      ]
     },
     "execution_count": 513,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweet.split()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 520,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[False]"
      ]
     },
     "execution_count": 520,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[query in tweet.split()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 521,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['RT',\n",
       " '@',\n",
       " 'lOR42wsOEFcv3f',\n",
       " ':',\n",
       " 'I',\n",
       " 'fall',\n",
       " 'too',\n",
       " 'fast',\n",
       " ',',\n",
       " 'crash',\n",
       " 'too',\n",
       " 'hard',\n",
       " ',',\n",
       " 'forgive',\n",
       " 'too',\n",
       " 'easily',\n",
       " 'and',\n",
       " 'care',\n",
       " 'too',\n",
       " 'much',\n",
       " '...',\n",
       " ':',\n",
       " '(',\n",
       " '#',\n",
       " 'amiright']"
      ]
     },
     "execution_count": 521,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nltk.word_tokenize(tweet)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 523,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[True]"
      ]
     },
     "execution_count": 523,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[query in nltk.word_tokenize(tweet)]\n",
    "# query"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 524,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['RT',\n",
       " '@',\n",
       " 'lOR42wsOEFcv3f',\n",
       " ':',\n",
       " 'I',\n",
       " 'fall',\n",
       " 'too',\n",
       " 'fast',\n",
       " ',',\n",
       " 'crash',\n",
       " 'too',\n",
       " 'hard',\n",
       " ',',\n",
       " 'forgive',\n",
       " 'too',\n",
       " 'easily',\n",
       " 'and',\n",
       " 'care',\n",
       " 'too',\n",
       " 'much',\n",
       " '...',\n",
       " ':',\n",
       " '(',\n",
       " '#',\n",
       " 'amiright']"
      ]
     },
     "execution_count": 524,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nltk.word_tokenize(tweet, language='spanish')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 531,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.tokenize import RegexpTokenizer\n",
    "custom_tokenizer = RegexpTokenizer('[a-zA-Z0-9]*', discard_empty=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 534,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from nltk.tokenize import "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 535,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['RT',\n",
       " 'lOR42wsOEFcv3f',\n",
       " 'I',\n",
       " 'fall',\n",
       " 'too',\n",
       " 'fast',\n",
       " 'crash',\n",
       " 'too',\n",
       " 'hard',\n",
       " 'forgive',\n",
       " 'too',\n",
       " 'easily',\n",
       " 'and',\n",
       " 'care',\n",
       " 'too',\n",
       " 'much',\n",
       " 'amiright']"
      ]
     },
     "execution_count": 535,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "custom_tokenizer.tokenize(tweet)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 536,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.tokenize import TweetTokenizer\n",
    "tweet_tokenizer = TweetTokenizer(strip_handles=True, reduce_len=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 537,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['RT',\n",
       " ':',\n",
       " 'I',\n",
       " 'fall',\n",
       " 'too',\n",
       " 'fast',\n",
       " ',',\n",
       " 'crash',\n",
       " 'too',\n",
       " 'hard',\n",
       " ',',\n",
       " 'forgive',\n",
       " 'too',\n",
       " 'easily',\n",
       " 'and',\n",
       " 'care',\n",
       " 'too',\n",
       " 'much',\n",
       " '...',\n",
       " ':(',\n",
       " '#amiright']"
      ]
     },
     "execution_count": 537,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweet_tokenizer.tokenize(tweet)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 288,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['RT',\n",
       " ':',\n",
       " 'I',\n",
       " 'fall',\n",
       " 'too_fast',\n",
       " ',',\n",
       " 'crash',\n",
       " 'too',\n",
       " 'hard',\n",
       " ',',\n",
       " 'forgive',\n",
       " 'too',\n",
       " 'easily',\n",
       " 'and',\n",
       " 'care',\n",
       " 'too',\n",
       " 'much',\n",
       " '...',\n",
       " ':(',\n",
       " '#amiright']"
      ]
     },
     "execution_count": 288,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from nltk.tokenize import MWETokenizer\n",
    "mwe = MWETokenizer()\n",
    "mwe.add_mwe(('too', 'fast'))\n",
    "mwe.tokenize(tweet_tokenizer.tokenize(tweet))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 540,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "add_mwe() takes 2 positional arguments but 3 were given",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-540-d652d5a4885d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mmwe\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_mwe\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'too'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'fast'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;34m'too'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'hard'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m: add_mwe() takes 2 positional arguments but 3 were given"
     ]
    }
   ],
   "source": [
    "mwe.add_mwe(('too', 'fast'), ('too', 'hard'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 539,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 539,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "query = 'fast'\n",
    "query in mwe.tokenize(tweet_tokenizer.tokenize(tweet))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Normalization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 541,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'rt @lor42wsoefcv3f: i fall too fast, crash too hard, forgive too easily and care too much... :( #amiright'"
      ]
     },
     "execution_count": 541,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweet.lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 542,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import string\n",
    "\n",
    "def normalize_tokens(tokenized_text):\n",
    "    # Lowercase\n",
    "    tokens = [t.lower() for t in tokenized_text]\n",
    "    # Remove hashtags\n",
    "    tokens = [t for t in tokens if not t.startswith('#')]\n",
    "    # Remove punctuation\n",
    "    tokens = [t for t in tokens if t not in string.punctuation]\n",
    "    # Keep only letters\n",
    "#     tokens = [t for t in tokens if re.match('^[a-z]+$', t)]\n",
    "    # Normalize characters\n",
    "#     tokens = [re.sub('á', 'a', t) for t in tokens]\n",
    "\n",
    "    return tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 543,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['muy', 'rápido']"
      ]
     },
     "execution_count": 543,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spanish_query = 'muy rápido'\n",
    "normalize_tokens(tweet_tokenizer.tokenize(spanish_query))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 508,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'muy rapido'"
      ]
     },
     "execution_count": 508,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import unidecode\n",
    "unidecode.unidecode(spanish_query)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 348,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['rt',\n",
       " 'i',\n",
       " 'fall',\n",
       " 'too',\n",
       " 'fast',\n",
       " 'crash',\n",
       " 'too',\n",
       " 'hard',\n",
       " 'forgive',\n",
       " 'too',\n",
       " 'easily',\n",
       " 'and',\n",
       " 'care',\n",
       " 'too',\n",
       " 'much',\n",
       " '...',\n",
       " ':(']"
      ]
     },
     "execution_count": 348,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "normalize_tokens(tweet_tokenizer.tokenize(tweet))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Uniform normalization principle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 545,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['too', 'fast', 'too', 'furious']"
      ]
     },
     "execution_count": 545,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "query = 'TOO fast TOO furious'\n",
    "tokenized_query = tweet_tokenizer.tokenize(query)\n",
    "normalized_query = normalize_tokens(tokenized_query)\n",
    "# normalized_query = tokenized_query\n",
    "normalized_query"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 546,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['rt',\n",
       " 'i',\n",
       " 'fall',\n",
       " 'too',\n",
       " 'fast',\n",
       " 'crash',\n",
       " 'too',\n",
       " 'hard',\n",
       " 'forgive',\n",
       " 'too',\n",
       " 'easily',\n",
       " 'and',\n",
       " 'care',\n",
       " 'too',\n",
       " 'much',\n",
       " '...',\n",
       " ':(']"
      ]
     },
     "execution_count": 546,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "normalized_tweet = normalize_tokens(tweet_tokenizer.tokenize(tweet))\n",
    "# normalized_tweet = normalize_tokens(tweet.split())\n",
    "normalized_tweet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 547,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'too', 'fast'}\n",
      "2 common word(s)\n"
     ]
    }
   ],
   "source": [
    "common_words = set(normalized_query).intersection(normalized_tweet)\n",
    "print(common_words)\n",
    "print(len(common_words), \"common word(s)\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Stopwords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 352,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to /home/ana/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 352,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nltk.download('stopwords')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 353,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['i',\n",
       " 'me',\n",
       " 'my',\n",
       " 'myself',\n",
       " 'we',\n",
       " 'our',\n",
       " 'ours',\n",
       " 'ourselves',\n",
       " 'you',\n",
       " \"you're\",\n",
       " \"you've\",\n",
       " \"you'll\",\n",
       " \"you'd\",\n",
       " 'your',\n",
       " 'yours',\n",
       " 'yourself',\n",
       " 'yourselves',\n",
       " 'he',\n",
       " 'him',\n",
       " 'his',\n",
       " 'himself',\n",
       " 'she',\n",
       " \"she's\",\n",
       " 'her',\n",
       " 'hers',\n",
       " 'herself',\n",
       " 'it',\n",
       " \"it's\",\n",
       " 'its',\n",
       " 'itself',\n",
       " 'they',\n",
       " 'them',\n",
       " 'their',\n",
       " 'theirs',\n",
       " 'themselves',\n",
       " 'what',\n",
       " 'which',\n",
       " 'who',\n",
       " 'whom',\n",
       " 'this',\n",
       " 'that',\n",
       " \"that'll\",\n",
       " 'these',\n",
       " 'those',\n",
       " 'am',\n",
       " 'is',\n",
       " 'are',\n",
       " 'was',\n",
       " 'were',\n",
       " 'be',\n",
       " 'been',\n",
       " 'being',\n",
       " 'have',\n",
       " 'has',\n",
       " 'had',\n",
       " 'having',\n",
       " 'do',\n",
       " 'does',\n",
       " 'did',\n",
       " 'doing',\n",
       " 'a',\n",
       " 'an',\n",
       " 'the',\n",
       " 'and',\n",
       " 'but',\n",
       " 'if',\n",
       " 'or',\n",
       " 'because',\n",
       " 'as',\n",
       " 'until',\n",
       " 'while',\n",
       " 'of',\n",
       " 'at',\n",
       " 'by',\n",
       " 'for',\n",
       " 'with',\n",
       " 'about',\n",
       " 'against',\n",
       " 'between',\n",
       " 'into',\n",
       " 'through',\n",
       " 'during',\n",
       " 'before',\n",
       " 'after',\n",
       " 'above',\n",
       " 'below',\n",
       " 'to',\n",
       " 'from',\n",
       " 'up',\n",
       " 'down',\n",
       " 'in',\n",
       " 'out',\n",
       " 'on',\n",
       " 'off',\n",
       " 'over',\n",
       " 'under',\n",
       " 'again',\n",
       " 'further',\n",
       " 'then',\n",
       " 'once',\n",
       " 'here',\n",
       " 'there',\n",
       " 'when',\n",
       " 'where',\n",
       " 'why',\n",
       " 'how',\n",
       " 'all',\n",
       " 'any',\n",
       " 'both',\n",
       " 'each',\n",
       " 'few',\n",
       " 'more',\n",
       " 'most',\n",
       " 'other',\n",
       " 'some',\n",
       " 'such',\n",
       " 'no',\n",
       " 'nor',\n",
       " 'not',\n",
       " 'only',\n",
       " 'own',\n",
       " 'same',\n",
       " 'so',\n",
       " 'than',\n",
       " 'too',\n",
       " 'very',\n",
       " 's',\n",
       " 't',\n",
       " 'can',\n",
       " 'will',\n",
       " 'just',\n",
       " 'don',\n",
       " \"don't\",\n",
       " 'should',\n",
       " \"should've\",\n",
       " 'now',\n",
       " 'd',\n",
       " 'll',\n",
       " 'm',\n",
       " 'o',\n",
       " 're',\n",
       " 've',\n",
       " 'y',\n",
       " 'ain',\n",
       " 'aren',\n",
       " \"aren't\",\n",
       " 'couldn',\n",
       " \"couldn't\",\n",
       " 'didn',\n",
       " \"didn't\",\n",
       " 'doesn',\n",
       " \"doesn't\",\n",
       " 'hadn',\n",
       " \"hadn't\",\n",
       " 'hasn',\n",
       " \"hasn't\",\n",
       " 'haven',\n",
       " \"haven't\",\n",
       " 'isn',\n",
       " \"isn't\",\n",
       " 'ma',\n",
       " 'mightn',\n",
       " \"mightn't\",\n",
       " 'mustn',\n",
       " \"mustn't\",\n",
       " 'needn',\n",
       " \"needn't\",\n",
       " 'shan',\n",
       " \"shan't\",\n",
       " 'shouldn',\n",
       " \"shouldn't\",\n",
       " 'wasn',\n",
       " \"wasn't\",\n",
       " 'weren',\n",
       " \"weren't\",\n",
       " 'won',\n",
       " \"won't\",\n",
       " 'wouldn',\n",
       " \"wouldn't\"]"
      ]
     },
     "execution_count": 353,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from nltk.corpus import stopwords\n",
    "stopwords.words('english')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 354,
   "metadata": {},
   "outputs": [],
   "source": [
    "blacklist_words = stopwords.words('english') + ['rt']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 355,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['fall', 'fast', 'crash', 'hard', 'forgive', 'easily', 'care', 'much', '...', ':(']\n"
     ]
    }
   ],
   "source": [
    "cleaned_tweet = [t for t in normalized_tweet if t not in blacklist_words]\n",
    "print(cleaned_tweet)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Stemming / Lemmatization\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 382,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.stem import PorterStemmer\n",
    "from nltk.stem import SnowballStemmer\n",
    "from nltk.stem import WordNetLemmatizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 387,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['fall',\n",
       " 'fast',\n",
       " 'crash',\n",
       " 'hard',\n",
       " 'forgiv',\n",
       " 'easili',\n",
       " 'care',\n",
       " 'much',\n",
       " '...',\n",
       " ':(']"
      ]
     },
     "execution_count": 387,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stemmer = PorterStemmer()\n",
    "\n",
    "[stemmer.stem(t) for t in cleaned_tweet]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 389,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['fall',\n",
       " 'fast',\n",
       " 'crash',\n",
       " 'hard',\n",
       " 'forgiv',\n",
       " 'easili',\n",
       " 'care',\n",
       " 'much',\n",
       " '...',\n",
       " ':(']"
      ]
     },
     "execution_count": 389,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stemmer = SnowballStemmer(language='english')\n",
    "\n",
    "[stemmer.stem(t) for t in cleaned_tweet]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 393,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['fall',\n",
       " 'fast',\n",
       " 'crash',\n",
       " 'hard',\n",
       " 'forgive',\n",
       " 'easily',\n",
       " 'care',\n",
       " 'much',\n",
       " '...',\n",
       " ':(']"
      ]
     },
     "execution_count": 393,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lemmatizer = WordNetLemmatizer()\n",
    "\n",
    "[lemmatizer.lemmatize(t) for t in cleaned_tweet]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 412,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('fall', 'NN'), ('fast', 'RB'), ('crash', 'JJ'), ('hard', 'JJ'), ('forgive', 'NN'), ('easily', 'RB'), ('care', 'VB'), ('much', 'JJ'), ('...', ':'), (':(', 'NN')]\n"
     ]
    }
   ],
   "source": [
    "tagged_tweet = nltk.pos_tag(cleaned_tweet)\n",
    "print(tagged_tweet)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 436,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.corpus import wordnet as wn\n",
    "tag_map = {'J': wn.ADJ, 'V': wn.VERB, 'R': wn.ADV, 'N': wn.NOUN}\n",
    "def get_lemmas(tokenized_text):\n",
    "    tagged_text = nltk.pos_tag(tokenized_text)\n",
    "    return [lemmatizer.lemmatize(w, pos=tag_map.get(p[0], wn.NOUN)) for (w, p) in tagged_text]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 443,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['the', 'fastest']\n"
     ]
    }
   ],
   "source": [
    "query = \"the fastest!\"\n",
    "normalized_query = normalize_tokens(tweet_tokenizer.tokenize(query))\n",
    "print(normalized_query)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 476,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['i', 'be', 'so', 'fast', 'i', 'be', 'the', 'fast']\n",
      "['the', 'fast']\n"
     ]
    }
   ],
   "source": [
    "lemmatized_tweet = get_lemmas(normalized_tweet)\n",
    "lemmatized_query = get_lemmas(normalized_query)\n",
    "print(lemmatized_tweet)\n",
    "print(lemmatized_query)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 551,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['i', 'be', 'so', 'fast', 'i', 'be', 'the', 'fast']"
      ]
     },
     "execution_count": 551,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweet = \"I am so fast, I am the fastest!\"\n",
    "normalized_tweet = normalize_tokens(tweet_tokenizer.tokenize(tweet))\n",
    "# normalized_tweet\n",
    "# [lemmatizer.lemmatize(t) for t in normalized_tweet]\n",
    "get_lemmas(normalized_tweet)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 477,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Common words: {'the', 'fast'}\n"
     ]
    }
   ],
   "source": [
    "print(\"Common words:\", set(lemmatized_tweet).intersection(set(lemmatized_query)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 554,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['fall',\n",
       " 'fast',\n",
       " 'crash',\n",
       " 'hard',\n",
       " 'forgive',\n",
       " 'easily',\n",
       " 'care',\n",
       " 'much',\n",
       " '...',\n",
       " ':(']"
      ]
     },
     "execution_count": 554,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_lemmas(cleaned_tweet)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Vocabulary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 555,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('be', 2), ('fast', 2), ('i', 2), ('so', 1), ('the', 1)]"
      ]
     },
     "execution_count": 555,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from collections import Counter\n",
    "\n",
    "Counter(get_lemmas(normalized_tweet)).most_common(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 466,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['i', 'be', 'so', 'fast', 'i', 'be', 'the', 'fast']\n"
     ]
    }
   ],
   "source": [
    "tweet = \"I am so fast, I am the fastest!\"\n",
    "normalized_tweet = normalize_tokens(tweet_tokenizer.tokenize(tweet))\n",
    "lemmatized_tweet = get_lemmas(normalized_tweet)\n",
    "print(lemmatized_tweet)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 468,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Counter({'i': 2, 'am': 2, 'fastest': 1, 'the': 1, 'so': 1, 'fast': 1})\n",
      "Counter({'be': 2, 'fast': 2, 'i': 2, 'so': 1, 'the': 1})\n"
     ]
    }
   ],
   "source": [
    "print(Counter(normalized_tweet))\n",
    "print(Counter(lemmatized_tweet))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Sentence segmentation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 556,
   "metadata": {},
   "outputs": [],
   "source": [
    "query = \"I am too fast. I am too furious.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 557,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.tokenize import sent_tokenize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 558,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['I am too fast.', 'I am too furious.']"
      ]
     },
     "execution_count": 558,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sent_tokenize(query)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 481,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Soy muy rápido!', 'Estoy muy furioso!']"
      ]
     },
     "execution_count": 481,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spanish_tokenizer = nltk.data.load('tokenizers/punkt/PY3/spanish.pickle')\n",
    "spanish_query = 'Soy muy rápido! Estoy muy furioso!'\n",
    "spanish_tokenizer.tokenize(spanish_query)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 482,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['J.K. Rowling is rich.', 'I am not as rich as J.K.']"
      ]
     },
     "execution_count": 482,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sent_tokenize(\"J.K. Rowling is rich. I am not as rich as J.K.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 559,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.tokenize import PunktSentenceTokenizer\n",
    "PunktSentenceTokenizer??"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Numericalization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 487,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['stay', 'tune']"
      ]
     },
     "execution_count": 487,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_lemmas(normalize_tokens(custom_tokenizer.tokenize(\"STAY TUNED!\")))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}