tokenization-spacy.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Spacy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Install Spacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: spacy in /home/ana/.local/lib/python3.5/site-packages\n",
      "Requirement already satisfied: pathlib in /home/ana/.local/lib/python3.5/site-packages (from spacy)\n",
      "Requirement already satisfied: numpy>=1.7 in /home/ana/.local/lib/python3.5/site-packages (from spacy)\n",
      "Requirement already satisfied: cymem<1.32,>=1.30 in /home/ana/.local/lib/python3.5/site-packages (from spacy)\n",
      "Requirement already satisfied: murmurhash<0.27,>=0.26 in /home/ana/.local/lib/python3.5/site-packages (from spacy)\n",
      "Requirement already satisfied: plac<1.0.0,>=0.9.6 in /home/ana/.local/lib/python3.5/site-packages (from spacy)\n",
      "Requirement already satisfied: preshed<2.0.0,>=1.0.0 in /home/ana/.local/lib/python3.5/site-packages (from spacy)\n",
      "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /home/ana/.local/lib/python3.5/site-packages (from spacy)\n",
      "Requirement already satisfied: ujson>=1.35 in /home/ana/.local/lib/python3.5/site-packages (from spacy)\n",
      "Requirement already satisfied: thinc<6.6.0,>=6.5.0 in /home/ana/.local/lib/python3.5/site-packages (from spacy)\n",
      "Requirement already satisfied: ftfy<5.0.0,>=4.4.2 in /home/ana/.local/lib/python3.5/site-packages (from spacy)\n",
      "Requirement already satisfied: dill<0.3,>=0.2 in /home/ana/.local/lib/python3.5/site-packages (from spacy)\n",
      "Requirement already satisfied: regex<2017.12.1,>=2017.4.1 in /home/ana/.local/lib/python3.5/site-packages (from spacy)\n",
      "Requirement already satisfied: six in /home/ana/.local/lib/python3.5/site-packages (from spacy)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /home/ana/.local/lib/python3.5/site-packages (from requests<3.0.0,>=2.13.0->spacy)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /home/ana/.local/lib/python3.5/site-packages (from requests<3.0.0,>=2.13.0->spacy)\n",
      "Requirement already satisfied: chardet<4,>=3.0.2 in /home/ana/.local/lib/python3.5/site-packages (from requests<3.0.0,>=2.13.0->spacy)\n",
      "Requirement already satisfied: idna<3,>=2.5 in /home/ana/.local/lib/python3.5/site-packages (from requests<3.0.0,>=2.13.0->spacy)\n",
      "Requirement already satisfied: tqdm<5.0.0,>=4.10.0 in /home/ana/.local/lib/python3.5/site-packages (from thinc<6.6.0,>=6.5.0->spacy)\n",
      "Requirement already satisfied: wrapt in /home/ana/.local/lib/python3.5/site-packages (from thinc<6.6.0,>=6.5.0->spacy)\n",
      "Requirement already satisfied: termcolor in /home/ana/.local/lib/python3.5/site-packages (from thinc<6.6.0,>=6.5.0->spacy)\n",
      "Requirement already satisfied: cytoolz<0.9,>=0.8 in /home/ana/.local/lib/python3.5/site-packages (from thinc<6.6.0,>=6.5.0->spacy)\n",
      "Requirement already satisfied: wcwidth in /home/ana/.local/lib/python3.5/site-packages (from ftfy<5.0.0,>=4.4.2->spacy)\n",
      "Requirement already satisfied: html5lib in /usr/lib/python3/dist-packages (from ftfy<5.0.0,>=4.4.2->spacy)\n",
      "Requirement already satisfied: toolz>=0.8.0 in /home/ana/.local/lib/python3.5/site-packages (from cytoolz<0.9,>=0.8->thinc<6.6.0,>=6.5.0->spacy)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You are using pip version 9.0.1, however version 21.0.1 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "pip install spacy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Download models or corpora"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-1.2.0/en_core_web_sm-1.2.0.tar.gz\n",
      "  Downloading https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-1.2.0/en_core_web_sm-1.2.0.tar.gz (52.2MB)\n",
      "  Requirement already satisfied (use --upgrade to upgrade): en-core-web-sm==1.2.0 from https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-1.2.0/en_core_web_sm-1.2.0.tar.gz in /home/ana/.local/lib/python3.5/site-packages\n",
      "Requirement already satisfied: spacy<2.0.0,>=1.7.0 in /home/ana/.local/lib/python3.5/site-packages (from en-core-web-sm==1.2.0)\n",
      "Requirement already satisfied: murmurhash<0.27,>=0.26 in /home/ana/.local/lib/python3.5/site-packages (from spacy<2.0.0,>=1.7.0->en-core-web-sm==1.2.0)\n",
      "Requirement already satisfied: ftfy<5.0.0,>=4.4.2 in /home/ana/.local/lib/python3.5/site-packages (from spacy<2.0.0,>=1.7.0->en-core-web-sm==1.2.0)\n",
      "Requirement already satisfied: pathlib in /home/ana/.local/lib/python3.5/site-packages (from spacy<2.0.0,>=1.7.0->en-core-web-sm==1.2.0)\n",
      "Requirement already satisfied: preshed<2.0.0,>=1.0.0 in /home/ana/.local/lib/python3.5/site-packages (from spacy<2.0.0,>=1.7.0->en-core-web-sm==1.2.0)\n",
      "Requirement already satisfied: ujson>=1.35 in /home/ana/.local/lib/python3.5/site-packages (from spacy<2.0.0,>=1.7.0->en-core-web-sm==1.2.0)\n",
      "Requirement already satisfied: plac<1.0.0,>=0.9.6 in /home/ana/.local/lib/python3.5/site-packages (from spacy<2.0.0,>=1.7.0->en-core-web-sm==1.2.0)\n",
      "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /home/ana/.local/lib/python3.5/site-packages (from spacy<2.0.0,>=1.7.0->en-core-web-sm==1.2.0)\n",
      "Requirement already satisfied: six in /home/ana/.local/lib/python3.5/site-packages (from spacy<2.0.0,>=1.7.0->en-core-web-sm==1.2.0)\n",
      "Requirement already satisfied: cymem<1.32,>=1.30 in /home/ana/.local/lib/python3.5/site-packages (from spacy<2.0.0,>=1.7.0->en-core-web-sm==1.2.0)\n",
      "Requirement already satisfied: regex<2017.12.1,>=2017.4.1 in /home/ana/.local/lib/python3.5/site-packages (from spacy<2.0.0,>=1.7.0->en-core-web-sm==1.2.0)\n",
      "Requirement already satisfied: thinc<6.6.0,>=6.5.0 in /home/ana/.local/lib/python3.5/site-packages (from spacy<2.0.0,>=1.7.0->en-core-web-sm==1.2.0)\n",
      "Requirement already satisfied: numpy>=1.7 in /home/ana/.local/lib/python3.5/site-packages (from spacy<2.0.0,>=1.7.0->en-core-web-sm==1.2.0)\n",
      "Requirement already satisfied: dill<0.3,>=0.2 in /home/ana/.local/lib/python3.5/site-packages (from spacy<2.0.0,>=1.7.0->en-core-web-sm==1.2.0)\n",
      "Requirement already satisfied: html5lib in /usr/lib/python3/dist-packages (from ftfy<5.0.0,>=4.4.2->spacy<2.0.0,>=1.7.0->en-core-web-sm==1.2.0)\n",
      "Requirement already satisfied: wcwidth in /home/ana/.local/lib/python3.5/site-packages (from ftfy<5.0.0,>=4.4.2->spacy<2.0.0,>=1.7.0->en-core-web-sm==1.2.0)\n",
      "Requirement already satisfied: chardet<4,>=3.0.2 in /home/ana/.local/lib/python3.5/site-packages (from requests<3.0.0,>=2.13.0->spacy<2.0.0,>=1.7.0->en-core-web-sm==1.2.0)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /home/ana/.local/lib/python3.5/site-packages (from requests<3.0.0,>=2.13.0->spacy<2.0.0,>=1.7.0->en-core-web-sm==1.2.0)\n",
      "Requirement already satisfied: idna<3,>=2.5 in /home/ana/.local/lib/python3.5/site-packages (from requests<3.0.0,>=2.13.0->spacy<2.0.0,>=1.7.0->en-core-web-sm==1.2.0)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /home/ana/.local/lib/python3.5/site-packages (from requests<3.0.0,>=2.13.0->spacy<2.0.0,>=1.7.0->en-core-web-sm==1.2.0)\n",
      "Requirement already satisfied: cytoolz<0.9,>=0.8 in /home/ana/.local/lib/python3.5/site-packages (from thinc<6.6.0,>=6.5.0->spacy<2.0.0,>=1.7.0->en-core-web-sm==1.2.0)\n",
      "Requirement already satisfied: tqdm<5.0.0,>=4.10.0 in /home/ana/.local/lib/python3.5/site-packages (from thinc<6.6.0,>=6.5.0->spacy<2.0.0,>=1.7.0->en-core-web-sm==1.2.0)\n",
      "Requirement already satisfied: wrapt in /home/ana/.local/lib/python3.5/site-packages (from thinc<6.6.0,>=6.5.0->spacy<2.0.0,>=1.7.0->en-core-web-sm==1.2.0)\n",
      "Requirement already satisfied: termcolor in /home/ana/.local/lib/python3.5/site-packages (from thinc<6.6.0,>=6.5.0->spacy<2.0.0,>=1.7.0->en-core-web-sm==1.2.0)\n",
      "Requirement already satisfied: toolz>=0.8.0 in /home/ana/.local/lib/python3.5/site-packages (from cytoolz<0.9,>=0.8->thinc<6.6.0,>=6.5.0->spacy<2.0.0,>=1.7.0->en-core-web-sm==1.2.0)\n",
      "\n",
      "    Downloading en_core_web_sm-1.2.0/en_core_web_sm-1.2.0.tar.gz\n",
      "\n",
      "\n",
      "\u001b[93m    Linking successful\u001b[0m\n",
      "\n",
      "    /home/ana/.local/lib/python3.5/site-packages/en_core_web_sm/en_core_web_sm-1.2.0\n",
      "    -->\n",
      "    /home/ana/.local/lib/python3.5/site-packages/spacy/data/en_core_web_sm\n",
      "\n",
      "    You can now load the model via spacy.load('en_core_web_sm').\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You are using pip version 9.0.1, however version 21.0.1 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "python3 -m spacy download en_core_web_sm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Define the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import spacy\n",
    "nlp = spacy.load(\"en_core_web_sm\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Process the texts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp = spacy.load(\"en\", disable=['ner'])\n",
    "\n",
    "\n",
    "def normalize(text, remove_stopwords, remove_punctuation):\n",
    "    text = text.lower()\n",
    "    text = nlp(text)\n",
    "    lemmatized = list()\n",
    "    for word in text:\n",
    "        if remove_stopwords and word.is_stop:\n",
    "            continue\n",
    "        if remove_punctuation and word.is_punct:\n",
    "            continue\n",
    "        lemma = word.lemma_.strip()\n",
    "        if lemma:\n",
    "            lemmatized.append(lemma)\n",
    "    return lemmatized\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['rt',\n",
       " '@lor42wsoefcv3f',\n",
       " 'fall',\n",
       " 'fast',\n",
       " 'crash',\n",
       " 'hard',\n",
       " 'forgive',\n",
       " 'easily',\n",
       " 'care',\n",
       " 'amiright']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweet = \"RT @lOR42wsOEFcv3f: I fall too fast, crash too hard, forgive too easily and care too much... :( #amiright\"\n",
    "normalize(tweet, remove_stopwords=True, remove_punctuation=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RT rt PROPN NNP compound XX True False\n",
      "@lOR42wsOEFcv3f @lor42wsoefcv3f NOUN NN nsubj @xXXddxxXXXxxdx False False\n",
      ": : PUNCT : punct : False False\n",
      "I -PRON- PRON PRP nsubj X True True\n",
      "fall fall VERB VBP acl xxxx True False\n",
      "too too ADV RB advmod xxx True True\n",
      "fast fast ADV RB advmod xxxx True False\n",
      ", , PUNCT , punct , False False\n",
      "crash crash NOUN NN acl xxxx True False\n",
      "too too ADV RB advmod xxx True True\n",
      "hard hard ADV RB acomp xxxx True False\n",
      ", , PUNCT , punct , False False\n",
      "forgive forgive VERB VB aux xxxx True False\n",
      "too too ADV RB advmod xxx True True\n",
      "easily easily ADV RB advmod xxxx True False\n",
      "and and CCONJ CC cc xxx True True\n",
      "care care VERB VB ROOT xxxx True False\n",
      "too too ADV RB advmod xxx True True\n",
      "much much ADV RB dobj xxxx True True\n",
      "... ... PUNCT NFP punct ... False False\n",
      ":( :( PUNCT NFP punct :( False False\n",
      "# # PUNCT NFP punct # False False\n",
      "amiright amiright ADV RB advmod xxxx True False\n"
     ]
    }
   ],
   "source": [
    "text = nlp(tweet)\n",
    "for token in text:\n",
    "    print(token.text, token.lemma_, token.pos_, token.tag_, token.dep_,\n",
    "            token.shape_, token.is_alpha, token.is_stop)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "from spacy.matcher import Matcher\n",
    "\n",
    "\n",
    "matcher = Matcher(nlp.vocab)\n",
    "matcher.add('HASHTAG', None, *[{'ORTH': '#'}, {'IS_ASCII': True}])\n",
    "\n",
    "doc = nlp('This is a #sentence. Here is another #hashtag. #The #End.')\n",
    "matches = matcher(doc)\n",
    "hashtags = []\n",
    "for match_id, start, end in matches:\n",
    "    hashtags.append(doc[start:end])\n",
    "\n",
    "for span in hashtags:\n",
    "    span.merge()\n",
    "\n",
    "print([t.text for t in doc])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}