03StopWords.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "03StopWords.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rd2Z5LOi4-j5",
        "colab_type": "text"
      },
      "source": [
        "Stop words are the **most frequently used words** in a language e.g. 'a', 'an', 'the'\n",
        "\n",
        "Stop words don't give any additionl information, so they can be filtered from the text to be processed\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ns8dMxmO5DYJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import spacy\n",
        "nlp = spacy.load('en_core_web_sm')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HTGMI6vW0svD",
        "colab_type": "code",
        "outputId": "0180f48e-1b07-4907-9f54-70b8520e6b47",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        }
      },
      "source": [
        "# Default stop words in SpaCy's (remember that sets are unordered):\n",
        "print(nlp.Defaults.stop_words)"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'sixty', 'one', 'there', 'again', 'unless', 'per', 'quite', 'so', 'however', 'hundred', 'then', 'except', 'where', 'thereupon', 'five', 'itself', 'could', 'those', 'no', 'thru', 'at', 'front', 'get', 'becomes', 'her', 'ca', '’ve', 'somewhere', 'thereby', 'throughout', 'none', 'would', \"'ll\", 'might', 'always', 'often', 'nine', 'twelve', 'n’t', 'towards', 'whereas', 'via', 'down', 'did', 'hers', 'three', 'toward', 'how', 'call', 'below', 'before', 'hereafter', 'seem', 'few', 'than', 'their', 'other', 'several', 'whoever', '’m', 'say', 'after', 'anything', 'both', 'been', 'least', 'through', \"'ve\", 'has', 'with', 'off', 'your', 'really', 'formerly', 'moreover', 'thence', 'onto', 'just', 'various', 'beforehand', 'do', 'hereupon', 'some', 'nevertheless', 'thereafter', '‘d', 'sometimes', 'who', 'either', 'yourselves', '‘ll', 'also', 'since', 'which', 'they', 'can', 'twenty', 'third', 'all', 'already', 'or', 'whether', 'take', 'these', 'four', 'whither', 'as', 'mostly', 'over', 'make', 'most', 'same', 'see', 'seeming', 'wherein', 'ourselves', 'anyhow', 'nowhere', 'even', 'whereafter', 'serious', 'here', 'around', 'anywhere', 'whence', 'less', 'latter', 'his', 'more', 'within', 'be', 'something', 'you', 'once', 'of', 'during', 'anyone', 'rather', 'out', 'made', 'still', 'keep', 'for', 'fifty', 'up', 'if', 'top', 'yourself', 'have', 'therefore', 'why', 'themselves', 'anyway', 'every', 'this', 'them', 'becoming', 'whereby', '’ll', 'another', 'empty', 'used', 'therein', 'above', 'yet', '‘m', 'noone', 'very', 'ours', 'become', 'him', 'he', 'seemed', 'we', 'last', 'nobody', 'although', 'am', 'amount', 'many', 'hereby', 'whose', 'to', 'became', 'when', 'was', 'whom', \"n't\", 'someone', 'thus', 'put', 'she', 'regarding', 'amongst', 'much', 'in', \"'re\", 'n‘t', 'using', 'together', 'is', 'himself', 'the', 'else', 'almost', 'full', 'next', 'besides', 'any', 'eleven', 'mine', 'somehow', 'others', 'though', 'under', 'only', 'never', 'whereupon', 'alone', 'that', 'namely', 'us', 'beyond', 'seems', 'its', 'please', 'first', 'part', '’s', 'being', 'yours', 'indeed', 'on', 'upon', 'well', 'what', '‘re', 'does', 'perhaps', 'neither', 'further', 'go', 'an', \"'d\", 'two', 'meanwhile', \"'m\", 'everyone', 'too', 'nor', '‘ve', 'bottom', '’d', 'name', 'enough', 'along', 'each', 'give', 'whole', 'ever', 'latterly', 'about', 'everywhere', 'move', 'wherever', 'six', 'my', 'eight', 'must', 'herself', 'done', 'from', 'by', 'myself', 'cannot', 'but', 'should', 'without', 'hence', 'because', 'afterwards', 'back', 'doing', '’re', 'former', 'will', 'whenever', 'fifteen', 'had', 'herein', 'me', 'into', 'across', 'elsewhere', 'forty', 'show', '‘s', 'side', 'i', 'beside', 'between', 'own', 'nothing', 'are', \"'s\", 'behind', 'otherwise', 'it', 'not', 're', 'until', 'whatever', 'due', 'while', 'were', 'and', 'our', 'ten', 'a', 'such', 'among', 'everything', 'may', 'sometime', 'against', 'now'}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aSMV0I2j3M7W",
        "colab_type": "code",
        "outputId": "ddd211a9-f4d0-48d6-e866-f7e8926842bb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "len(nlp.Defaults.stop_words) # to check total number of stop words in Spacy's"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "326"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UH38qq701VIH",
        "colab_type": "code",
        "outputId": "2163b90c-2df8-435f-8452-5c93813f6704",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "nlp.vocab['always'].is_stop # to check wheather a word is a stop word or not"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uHftlLsU1eWA",
        "colab_type": "code",
        "outputId": "75342194-6089-4fd8-a410-afb47146a685",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "nlp.vocab['Finance'].is_stop"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "False"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "E2FeX-a44Jeh",
        "colab_type": "text"
      },
      "source": [
        "# 1) To add a stop word"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dBDgviYz4L-c",
        "colab_type": "code",
        "outputId": "2d24b0fa-b3d6-4e75-9f87-f2943177b253",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "nlp.vocab['asdf'].is_stop"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "False"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wbkb2IXQ4P71",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "nlp.Defaults.stop_words.add('asdf')\n",
        "\n",
        "nlp.vocab['asdf'].is_stop = True  # Set stop_word tag"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k-aI_PQT4R7k",
        "colab_type": "code",
        "outputId": "01f90587-d8bd-4d6f-cb34-5a276a49337d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "nlp.vocab['asdf'].is_stop"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EtBghkJs40AD",
        "colab_type": "text"
      },
      "source": [
        "# 2) To remove a stop word"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vDQlYKzw51G9",
        "colab_type": "code",
        "outputId": "274a7092-b2f6-4a55-88f8-83acce246f03",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "nlp.vocab['latter'].is_stop"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ye5Lc3C941p2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "nlp.Defaults.stop_words.remove('latter')\n",
        "\n",
        "nlp.vocab['latter'].is_stop = False"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JurlzeGi5cwr",
        "colab_type": "code",
        "outputId": "52c86e98-9ff9-40f1-ebd5-dba093a5422d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "nlp.vocab['latter'].is_stop"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "False"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    }
  ]
}