text_preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import re\n",
    "import string\n",
    "import spacy\n",
    "import emoji"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Read csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>author_id</th>\n",
       "      <th>inbound</th>\n",
       "      <th>created_at</th>\n",
       "      <th>text</th>\n",
       "      <th>response_tweet_id</th>\n",
       "      <th>in_response_to_tweet_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>119237</td>\n",
       "      <td>105834</td>\n",
       "      <td>True</td>\n",
       "      <td>Wed Oct 11 06:55:44 +0000 2017</td>\n",
       "      <td>@AppleSupport causing the reply to be disregar...</td>\n",
       "      <td>119236</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>119238</td>\n",
       "      <td>ChaseSupport</td>\n",
       "      <td>False</td>\n",
       "      <td>Wed Oct 11 13:25:49 +0000 2017</td>\n",
       "      <td>@105835 Your business means a lot to us. Pleas...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>119239.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>119239</td>\n",
       "      <td>105835</td>\n",
       "      <td>True</td>\n",
       "      <td>Wed Oct 11 13:00:09 +0000 2017</td>\n",
       "      <td>@76328 I really hope you all change but I'm su...</td>\n",
       "      <td>119238</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>119240</td>\n",
       "      <td>VirginTrains</td>\n",
       "      <td>False</td>\n",
       "      <td>Tue Oct 10 15:16:08 +0000 2017</td>\n",
       "      <td>@105836 LiveChat is online at the moment - htt...</td>\n",
       "      <td>119241</td>\n",
       "      <td>119242.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>119241</td>\n",
       "      <td>105836</td>\n",
       "      <td>True</td>\n",
       "      <td>Tue Oct 10 15:17:21 +0000 2017</td>\n",
       "      <td>@VirginTrains see attached error message. I've...</td>\n",
       "      <td>119243</td>\n",
       "      <td>119240.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   tweet_id     author_id  inbound                      created_at  \\\n",
       "0    119237        105834     True  Wed Oct 11 06:55:44 +0000 2017   \n",
       "1    119238  ChaseSupport    False  Wed Oct 11 13:25:49 +0000 2017   \n",
       "2    119239        105835     True  Wed Oct 11 13:00:09 +0000 2017   \n",
       "3    119240  VirginTrains    False  Tue Oct 10 15:16:08 +0000 2017   \n",
       "4    119241        105836     True  Tue Oct 10 15:17:21 +0000 2017   \n",
       "\n",
       "                                                text response_tweet_id  \\\n",
       "0  @AppleSupport causing the reply to be disregar...            119236   \n",
       "1  @105835 Your business means a lot to us. Pleas...               NaN   \n",
       "2  @76328 I really hope you all change but I'm su...            119238   \n",
       "3  @105836 LiveChat is online at the moment - htt...            119241   \n",
       "4  @VirginTrains see attached error message. I've...            119243   \n",
       "\n",
       "   in_response_to_tweet_id  \n",
       "0                      NaN  \n",
       "1                 119239.0  \n",
       "2                      NaN  \n",
       "3                 119242.0  \n",
       "4                 119240.0  "
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv(r'sample.csv')\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### lower case"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "def to_lower(text):\n",
    "    return text.lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>author_id</th>\n",
       "      <th>inbound</th>\n",
       "      <th>created_at</th>\n",
       "      <th>text</th>\n",
       "      <th>response_tweet_id</th>\n",
       "      <th>in_response_to_tweet_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>119237</td>\n",
       "      <td>105834</td>\n",
       "      <td>True</td>\n",
       "      <td>Wed Oct 11 06:55:44 +0000 2017</td>\n",
       "      <td>@applesupport causing the reply to be disregar...</td>\n",
       "      <td>119236</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>119238</td>\n",
       "      <td>ChaseSupport</td>\n",
       "      <td>False</td>\n",
       "      <td>Wed Oct 11 13:25:49 +0000 2017</td>\n",
       "      <td>@105835 your business means a lot to us. pleas...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>119239.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>119239</td>\n",
       "      <td>105835</td>\n",
       "      <td>True</td>\n",
       "      <td>Wed Oct 11 13:00:09 +0000 2017</td>\n",
       "      <td>@76328 i really hope you all change but i'm su...</td>\n",
       "      <td>119238</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   tweet_id     author_id  inbound                      created_at  \\\n",
       "0    119237        105834     True  Wed Oct 11 06:55:44 +0000 2017   \n",
       "1    119238  ChaseSupport    False  Wed Oct 11 13:25:49 +0000 2017   \n",
       "2    119239        105835     True  Wed Oct 11 13:00:09 +0000 2017   \n",
       "\n",
       "                                                text response_tweet_id  \\\n",
       "0  @applesupport causing the reply to be disregar...            119236   \n",
       "1  @105835 your business means a lot to us. pleas...               NaN   \n",
       "2  @76328 i really hope you all change but i'm su...            119238   \n",
       "\n",
       "   in_response_to_tweet_id  \n",
       "0                      NaN  \n",
       "1                 119239.0  \n",
       "2                      NaN  "
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['text'] = data['text'].apply(to_lower)\n",
    "data[:3]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### remove html tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_html_tags(text):\n",
    "    pattern = re.compile('<.*?>')\n",
    "    return pattern.sub('',text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>author_id</th>\n",
       "      <th>inbound</th>\n",
       "      <th>created_at</th>\n",
       "      <th>text</th>\n",
       "      <th>response_tweet_id</th>\n",
       "      <th>in_response_to_tweet_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>119237</td>\n",
       "      <td>105834</td>\n",
       "      <td>True</td>\n",
       "      <td>Wed Oct 11 06:55:44 +0000 2017</td>\n",
       "      <td>@applesupport causing the reply to be disregar...</td>\n",
       "      <td>119236</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>119238</td>\n",
       "      <td>ChaseSupport</td>\n",
       "      <td>False</td>\n",
       "      <td>Wed Oct 11 13:25:49 +0000 2017</td>\n",
       "      <td>@105835 your business means a lot to us. pleas...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>119239.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>119239</td>\n",
       "      <td>105835</td>\n",
       "      <td>True</td>\n",
       "      <td>Wed Oct 11 13:00:09 +0000 2017</td>\n",
       "      <td>@76328 i really hope you all change but i'm su...</td>\n",
       "      <td>119238</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   tweet_id     author_id  inbound                      created_at  \\\n",
       "0    119237        105834     True  Wed Oct 11 06:55:44 +0000 2017   \n",
       "1    119238  ChaseSupport    False  Wed Oct 11 13:25:49 +0000 2017   \n",
       "2    119239        105835     True  Wed Oct 11 13:00:09 +0000 2017   \n",
       "\n",
       "                                                text response_tweet_id  \\\n",
       "0  @applesupport causing the reply to be disregar...            119236   \n",
       "1  @105835 your business means a lot to us. pleas...               NaN   \n",
       "2  @76328 i really hope you all change but i'm su...            119238   \n",
       "\n",
       "   in_response_to_tweet_id  \n",
       "0                      NaN  \n",
       "1                 119239.0  \n",
       "2                      NaN  "
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['text']=data['text'].apply(remove_html_tags)\n",
    "data[:3]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### remove url"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_url(text):\n",
    "    pattern = re.compile(r\"https://\\S+|www\\.S+\")\n",
    "    return pattern.sub('',text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>author_id</th>\n",
       "      <th>inbound</th>\n",
       "      <th>created_at</th>\n",
       "      <th>text</th>\n",
       "      <th>response_tweet_id</th>\n",
       "      <th>in_response_to_tweet_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>119237</td>\n",
       "      <td>105834</td>\n",
       "      <td>True</td>\n",
       "      <td>Wed Oct 11 06:55:44 +0000 2017</td>\n",
       "      <td>@applesupport causing the reply to be disregar...</td>\n",
       "      <td>119236</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>119238</td>\n",
       "      <td>ChaseSupport</td>\n",
       "      <td>False</td>\n",
       "      <td>Wed Oct 11 13:25:49 +0000 2017</td>\n",
       "      <td>@105835 your business means a lot to us. pleas...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>119239.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>119239</td>\n",
       "      <td>105835</td>\n",
       "      <td>True</td>\n",
       "      <td>Wed Oct 11 13:00:09 +0000 2017</td>\n",
       "      <td>@76328 i really hope you all change but i'm su...</td>\n",
       "      <td>119238</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>119240</td>\n",
       "      <td>VirginTrains</td>\n",
       "      <td>False</td>\n",
       "      <td>Tue Oct 10 15:16:08 +0000 2017</td>\n",
       "      <td>@105836 livechat is online at the moment -  or...</td>\n",
       "      <td>119241</td>\n",
       "      <td>119242.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>119241</td>\n",
       "      <td>105836</td>\n",
       "      <td>True</td>\n",
       "      <td>Tue Oct 10 15:17:21 +0000 2017</td>\n",
       "      <td>@virgintrains see attached error message. i've...</td>\n",
       "      <td>119243</td>\n",
       "      <td>119240.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>119243</td>\n",
       "      <td>VirginTrains</td>\n",
       "      <td>False</td>\n",
       "      <td>Tue Oct 10 15:25:14 +0000 2017</td>\n",
       "      <td>@105836 have you tried from another device, mi...</td>\n",
       "      <td>119244</td>\n",
       "      <td>119241.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>119244</td>\n",
       "      <td>105836</td>\n",
       "      <td>True</td>\n",
       "      <td>Tue Oct 10 15:26:44 +0000 2017</td>\n",
       "      <td>@virgintrains yep, i've tried laptop too sever...</td>\n",
       "      <td>119245</td>\n",
       "      <td>119243.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   tweet_id     author_id  inbound                      created_at  \\\n",
       "0    119237        105834     True  Wed Oct 11 06:55:44 +0000 2017   \n",
       "1    119238  ChaseSupport    False  Wed Oct 11 13:25:49 +0000 2017   \n",
       "2    119239        105835     True  Wed Oct 11 13:00:09 +0000 2017   \n",
       "3    119240  VirginTrains    False  Tue Oct 10 15:16:08 +0000 2017   \n",
       "4    119241        105836     True  Tue Oct 10 15:17:21 +0000 2017   \n",
       "5    119243  VirginTrains    False  Tue Oct 10 15:25:14 +0000 2017   \n",
       "6    119244        105836     True  Tue Oct 10 15:26:44 +0000 2017   \n",
       "\n",
       "                                                text response_tweet_id  \\\n",
       "0  @applesupport causing the reply to be disregar...            119236   \n",
       "1  @105835 your business means a lot to us. pleas...               NaN   \n",
       "2  @76328 i really hope you all change but i'm su...            119238   \n",
       "3  @105836 livechat is online at the moment -  or...            119241   \n",
       "4  @virgintrains see attached error message. i've...            119243   \n",
       "5  @105836 have you tried from another device, mi...            119244   \n",
       "6  @virgintrains yep, i've tried laptop too sever...            119245   \n",
       "\n",
       "   in_response_to_tweet_id  \n",
       "0                      NaN  \n",
       "1                 119239.0  \n",
       "2                      NaN  \n",
       "3                 119242.0  \n",
       "4                 119240.0  \n",
       "5                 119241.0  \n",
       "6                 119243.0  "
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['text'] = data['text'].apply(remove_url)\n",
    "data[:7]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### string manipulation for removing punctuations ('!\"#$%&\\'()*+,-./:;<=>?@[\\\\]^_`{|}~')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_punctuations(text):\n",
    "    exclusion_list = string.punctuation\n",
    "    for chars in exclusion_list:\n",
    "        text = text.replace(chars,'')\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['text'] = data['text'].apply(remove_punctuations)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### expand Abbreviations [e.g. ASAP : As Soon as Possible]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{' AFAIK': 'As Far As I Know',\n",
       " 'AFK': 'Away From Keyboard',\n",
       " 'ASAP': 'As Soon As Possible',\n",
       " 'BTW': 'By The Way',\n",
       " 'B4': 'Before',\n",
       " 'LMAO': 'Laugh My A.. Off',\n",
       " 'FYI': 'For your information'}"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "chat_words={\n",
    "   \" AFAIK\":\"As Far As I Know\",\n",
    "\"AFK\": \"Away From Keyboard\",\n",
    "\"ASAP\":\"As Soon As Possible\",\n",
    "\"BTW\":\"By The Way\",\n",
    "\"B4\":\"Before\",\n",
    "\"LMAO\":\"Laugh My A.. Off\",\n",
    "\"FYI\":\"For your information\"\n",
    "}\n",
    "chat_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "def chat_conversion(text):\n",
    "    new_txt=[]\n",
    "    for w in text.split():\n",
    "        if w.upper()in chat_words:\n",
    "            new_txt.append(chat_words[w.upper()])\n",
    "        else:\n",
    "            new_txt.append(w)\n",
    "    return ' '.join(new_txt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>author_id</th>\n",
       "      <th>inbound</th>\n",
       "      <th>created_at</th>\n",
       "      <th>text</th>\n",
       "      <th>response_tweet_id</th>\n",
       "      <th>in_response_to_tweet_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>119237</td>\n",
       "      <td>105834</td>\n",
       "      <td>True</td>\n",
       "      <td>Wed Oct 11 06:55:44 +0000 2017</td>\n",
       "      <td>applesupport causing the reply to be disregard...</td>\n",
       "      <td>119236</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>119238</td>\n",
       "      <td>ChaseSupport</td>\n",
       "      <td>False</td>\n",
       "      <td>Wed Oct 11 13:25:49 +0000 2017</td>\n",
       "      <td>105835 your business means a lot to us please ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>119239.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>119239</td>\n",
       "      <td>105835</td>\n",
       "      <td>True</td>\n",
       "      <td>Wed Oct 11 13:00:09 +0000 2017</td>\n",
       "      <td>76328 i really hope you all change but im sure...</td>\n",
       "      <td>119238</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>119240</td>\n",
       "      <td>VirginTrains</td>\n",
       "      <td>False</td>\n",
       "      <td>Tue Oct 10 15:16:08 +0000 2017</td>\n",
       "      <td>105836 livechat is online at the moment or con...</td>\n",
       "      <td>119241</td>\n",
       "      <td>119242.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>119241</td>\n",
       "      <td>105836</td>\n",
       "      <td>True</td>\n",
       "      <td>Tue Oct 10 15:17:21 +0000 2017</td>\n",
       "      <td>virgintrains see attached error message ive tr...</td>\n",
       "      <td>119243</td>\n",
       "      <td>119240.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88</th>\n",
       "      <td>119330</td>\n",
       "      <td>105859</td>\n",
       "      <td>True</td>\n",
       "      <td>Wed Oct 11 13:50:42 +0000 2017</td>\n",
       "      <td>105860 i wish amazon had an option of where i ...</td>\n",
       "      <td>119329</td>\n",
       "      <td>119331.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89</th>\n",
       "      <td>119331</td>\n",
       "      <td>105860</td>\n",
       "      <td>True</td>\n",
       "      <td>Wed Oct 11 13:47:14 +0000 2017</td>\n",
       "      <td>they reschedule my shit for tomorrow</td>\n",
       "      <td>119330</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>119332</td>\n",
       "      <td>Tesco</td>\n",
       "      <td>False</td>\n",
       "      <td>Wed Oct 11 13:34:06 +0000 2017</td>\n",
       "      <td>105861 hey sara sorry to hear of the issues yo...</td>\n",
       "      <td>119333</td>\n",
       "      <td>119334.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91</th>\n",
       "      <td>119333</td>\n",
       "      <td>105861</td>\n",
       "      <td>True</td>\n",
       "      <td>Wed Oct 11 14:05:18 +0000 2017</td>\n",
       "      <td>tesco bit of both finding the layout cumbersom...</td>\n",
       "      <td>119335,119336</td>\n",
       "      <td>119332.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92</th>\n",
       "      <td>119335</td>\n",
       "      <td>Tesco</td>\n",
       "      <td>False</td>\n",
       "      <td>Wed Oct 11 15:38:07 +0000 2017</td>\n",
       "      <td>105861 if that doesnt help please dm your full...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>119333.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>93 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    tweet_id     author_id  inbound                      created_at  \\\n",
       "0     119237        105834     True  Wed Oct 11 06:55:44 +0000 2017   \n",
       "1     119238  ChaseSupport    False  Wed Oct 11 13:25:49 +0000 2017   \n",
       "2     119239        105835     True  Wed Oct 11 13:00:09 +0000 2017   \n",
       "3     119240  VirginTrains    False  Tue Oct 10 15:16:08 +0000 2017   \n",
       "4     119241        105836     True  Tue Oct 10 15:17:21 +0000 2017   \n",
       "..       ...           ...      ...                             ...   \n",
       "88    119330        105859     True  Wed Oct 11 13:50:42 +0000 2017   \n",
       "89    119331        105860     True  Wed Oct 11 13:47:14 +0000 2017   \n",
       "90    119332         Tesco    False  Wed Oct 11 13:34:06 +0000 2017   \n",
       "91    119333        105861     True  Wed Oct 11 14:05:18 +0000 2017   \n",
       "92    119335         Tesco    False  Wed Oct 11 15:38:07 +0000 2017   \n",
       "\n",
       "                                                 text response_tweet_id  \\\n",
       "0   applesupport causing the reply to be disregard...            119236   \n",
       "1   105835 your business means a lot to us please ...               NaN   \n",
       "2   76328 i really hope you all change but im sure...            119238   \n",
       "3   105836 livechat is online at the moment or con...            119241   \n",
       "4   virgintrains see attached error message ive tr...            119243   \n",
       "..                                                ...               ...   \n",
       "88  105860 i wish amazon had an option of where i ...            119329   \n",
       "89               they reschedule my shit for tomorrow            119330   \n",
       "90  105861 hey sara sorry to hear of the issues yo...            119333   \n",
       "91  tesco bit of both finding the layout cumbersom...     119335,119336   \n",
       "92  105861 if that doesnt help please dm your full...               NaN   \n",
       "\n",
       "    in_response_to_tweet_id  \n",
       "0                       NaN  \n",
       "1                  119239.0  \n",
       "2                       NaN  \n",
       "3                  119242.0  \n",
       "4                  119240.0  \n",
       "..                      ...  \n",
       "88                 119331.0  \n",
       "89                      NaN  \n",
       "90                 119334.0  \n",
       "91                 119332.0  \n",
       "92                 119333.0  \n",
       "\n",
       "[93 rows x 7 columns]"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['text'] = data['text'].apply(chat_conversion)\n",
    "data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### spelling Correction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp = spacy.load(\"en_core_web_sm\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "from spellchecker import SpellChecker\n",
    "\n",
    "# Create a SpellChecker instance\n",
    "spell = SpellChecker()\n",
    "\n",
    "# Define a function for spelling correction using SpaCy and pyspellchecker\n",
    "def correct_spelling(text):\n",
    "    # Tokenize the text using SpaCy\n",
    "    doc = nlp(text)\n",
    "\n",
    "    # Get the tokens and their corrected spellings\n",
    "    corrected_tokens = [spell.correction(token.text) for token in doc]\n",
    "\n",
    "    # Join the corrected tokens back into a string\n",
    "    corrected_text = ' '.join(corrected_tokens)\n",
    "\n",
    "    return corrected_text\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### remove stopwords using SPACY"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original Text: This is an example sentence with some stopwords.\n",
      "Text without Stopwords: example sentence stopwords .\n"
     ]
    }
   ],
   "source": [
    "import spacy\n",
    "\n",
    "# Load the SpaCy English model\n",
    "nlp = spacy.load(\"en_core_web_sm\")\n",
    "\n",
    "# Define a function to remove stopwords\n",
    "def remove_stopwords(text):\n",
    "    # Tokenize the text using SpaCy\n",
    "    doc = nlp(text)\n",
    "\n",
    "    # Filter out tokens that are not stop words\n",
    "    filtered_tokens = [token.text for token in doc if not token.is_stop]\n",
    "\n",
    "    # Join the filtered tokens back into a string\n",
    "    result_text = ' '.join(filtered_tokens)\n",
    "\n",
    "    return result_text\n",
    "\n",
    "# Example usage\n",
    "text_with_stopwords = \"This is an example sentence with some stopwords.\"\n",
    "text_without_stopwords = remove_stopwords(text_with_stopwords)\n",
    "\n",
    "print(\"Original Text:\", text_with_stopwords)\n",
    "print(\"Text without Stopwords:\", text_without_stopwords)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>author_id</th>\n",
       "      <th>inbound</th>\n",
       "      <th>created_at</th>\n",
       "      <th>text</th>\n",
       "      <th>response_tweet_id</th>\n",
       "      <th>in_response_to_tweet_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>119237</td>\n",
       "      <td>105834</td>\n",
       "      <td>True</td>\n",
       "      <td>Wed Oct 11 06:55:44 +0000 2017</td>\n",
       "      <td>applesupport causing reply disregarded tapped ...</td>\n",
       "      <td>119236</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>119238</td>\n",
       "      <td>ChaseSupport</td>\n",
       "      <td>False</td>\n",
       "      <td>Wed Oct 11 13:25:49 +0000 2017</td>\n",
       "      <td>105835 business means lot dm zip code addition...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>119239.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>119239</td>\n",
       "      <td>105835</td>\n",
       "      <td>True</td>\n",
       "      <td>Wed Oct 11 13:00:09 +0000 2017</td>\n",
       "      <td>76328 hope change m sure wo nt nt</td>\n",
       "      <td>119238</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>119240</td>\n",
       "      <td>VirginTrains</td>\n",
       "      <td>False</td>\n",
       "      <td>Tue Oct 10 15:16:08 +0000 2017</td>\n",
       "      <td>105836 livechat online moment contact 03331 03...</td>\n",
       "      <td>119241</td>\n",
       "      <td>119242.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>119241</td>\n",
       "      <td>105836</td>\n",
       "      <td>True</td>\n",
       "      <td>Tue Oct 10 15:17:21 +0000 2017</td>\n",
       "      <td>virgintrains attached error message ve tried l...</td>\n",
       "      <td>119243</td>\n",
       "      <td>119240.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   tweet_id     author_id  inbound                      created_at  \\\n",
       "0    119237        105834     True  Wed Oct 11 06:55:44 +0000 2017   \n",
       "1    119238  ChaseSupport    False  Wed Oct 11 13:25:49 +0000 2017   \n",
       "2    119239        105835     True  Wed Oct 11 13:00:09 +0000 2017   \n",
       "3    119240  VirginTrains    False  Tue Oct 10 15:16:08 +0000 2017   \n",
       "4    119241        105836     True  Tue Oct 10 15:17:21 +0000 2017   \n",
       "\n",
       "                                                text response_tweet_id  \\\n",
       "0  applesupport causing reply disregarded tapped ...            119236   \n",
       "1  105835 business means lot dm zip code addition...               NaN   \n",
       "2                  76328 hope change m sure wo nt nt            119238   \n",
       "3  105836 livechat online moment contact 03331 03...            119241   \n",
       "4  virgintrains attached error message ve tried l...            119243   \n",
       "\n",
       "   in_response_to_tweet_id  \n",
       "0                      NaN  \n",
       "1                 119239.0  \n",
       "2                      NaN  \n",
       "3                 119242.0  \n",
       "4                 119240.0  "
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['text']=data['text'].apply(remove_stopwords)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### remove EMOJIs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_emoji(text):\n",
    "    clean_text=emoji.demojize(text)\n",
    "    return clean_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>author_id</th>\n",
       "      <th>inbound</th>\n",
       "      <th>created_at</th>\n",
       "      <th>text</th>\n",
       "      <th>response_tweet_id</th>\n",
       "      <th>in_response_to_tweet_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>119237</td>\n",
       "      <td>105834</td>\n",
       "      <td>True</td>\n",
       "      <td>Wed Oct 11 06:55:44 +0000 2017</td>\n",
       "      <td>applesupport causing reply disregarded tapped ...</td>\n",
       "      <td>119236</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>119238</td>\n",
       "      <td>ChaseSupport</td>\n",
       "      <td>False</td>\n",
       "      <td>Wed Oct 11 13:25:49 +0000 2017</td>\n",
       "      <td>105835 business means lot dm zip code addition...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>119239.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>119239</td>\n",
       "      <td>105835</td>\n",
       "      <td>True</td>\n",
       "      <td>Wed Oct 11 13:00:09 +0000 2017</td>\n",
       "      <td>76328 hope change m sure wo nt nt</td>\n",
       "      <td>119238</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>119240</td>\n",
       "      <td>VirginTrains</td>\n",
       "      <td>False</td>\n",
       "      <td>Tue Oct 10 15:16:08 +0000 2017</td>\n",
       "      <td>105836 livechat online moment contact 03331 03...</td>\n",
       "      <td>119241</td>\n",
       "      <td>119242.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>119241</td>\n",
       "      <td>105836</td>\n",
       "      <td>True</td>\n",
       "      <td>Tue Oct 10 15:17:21 +0000 2017</td>\n",
       "      <td>virgintrains attached error message ve tried l...</td>\n",
       "      <td>119243</td>\n",
       "      <td>119240.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   tweet_id     author_id  inbound                      created_at  \\\n",
       "0    119237        105834     True  Wed Oct 11 06:55:44 +0000 2017   \n",
       "1    119238  ChaseSupport    False  Wed Oct 11 13:25:49 +0000 2017   \n",
       "2    119239        105835     True  Wed Oct 11 13:00:09 +0000 2017   \n",
       "3    119240  VirginTrains    False  Tue Oct 10 15:16:08 +0000 2017   \n",
       "4    119241        105836     True  Tue Oct 10 15:17:21 +0000 2017   \n",
       "\n",
       "                                                text response_tweet_id  \\\n",
       "0  applesupport causing reply disregarded tapped ...            119236   \n",
       "1  105835 business means lot dm zip code addition...               NaN   \n",
       "2                  76328 hope change m sure wo nt nt            119238   \n",
       "3  105836 livechat online moment contact 03331 03...            119241   \n",
       "4  virgintrains attached error message ve tried l...            119243   \n",
       "\n",
       "   in_response_to_tweet_id  \n",
       "0                      NaN  \n",
       "1                 119239.0  \n",
       "2                      NaN  \n",
       "3                 119242.0  \n",
       "4                 119240.0  "
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['text']=data['text'].apply(remove_emoji)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Lemmatization using SPACY "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lemmatize_text(text):\n",
    "    # Process the text using SpaCy\n",
    "    doc = nlp(text)\n",
    "    \n",
    "    # Extract the lemmatized forms of each token\n",
    "    lemmatized_tokens = [token.lemma_ for token in doc]\n",
    "    \n",
    "    # Join the lemmatized tokens back into a string\n",
    "    result_text = ' '.join(lemmatized_tokens)\n",
    "    \n",
    "    return result_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>author_id</th>\n",
       "      <th>inbound</th>\n",
       "      <th>created_at</th>\n",
       "      <th>text</th>\n",
       "      <th>response_tweet_id</th>\n",
       "      <th>in_response_to_tweet_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>119237</td>\n",
       "      <td>105834</td>\n",
       "      <td>True</td>\n",
       "      <td>Wed Oct 11 06:55:44 +0000 2017</td>\n",
       "      <td>applesupport cause reply disregard tap notific...</td>\n",
       "      <td>119236</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>119238</td>\n",
       "      <td>ChaseSupport</td>\n",
       "      <td>False</td>\n",
       "      <td>Wed Oct 11 13:25:49 +0000 2017</td>\n",
       "      <td>105835 business mean lot dm zip code additiona...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>119239.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>119239</td>\n",
       "      <td>105835</td>\n",
       "      <td>True</td>\n",
       "      <td>Wed Oct 11 13:00:09 +0000 2017</td>\n",
       "      <td>76328 hope change m sure will not not</td>\n",
       "      <td>119238</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>119240</td>\n",
       "      <td>VirginTrains</td>\n",
       "      <td>False</td>\n",
       "      <td>Tue Oct 10 15:16:08 +0000 2017</td>\n",
       "      <td>105836 livechat online moment contact 03331 03...</td>\n",
       "      <td>119241</td>\n",
       "      <td>119242.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>119241</td>\n",
       "      <td>105836</td>\n",
       "      <td>True</td>\n",
       "      <td>Tue Oct 10 15:17:21 +0000 2017</td>\n",
       "      <td>virgintrain attach error message ve try leave ...</td>\n",
       "      <td>119243</td>\n",
       "      <td>119240.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   tweet_id     author_id  inbound                      created_at  \\\n",
       "0    119237        105834     True  Wed Oct 11 06:55:44 +0000 2017   \n",
       "1    119238  ChaseSupport    False  Wed Oct 11 13:25:49 +0000 2017   \n",
       "2    119239        105835     True  Wed Oct 11 13:00:09 +0000 2017   \n",
       "3    119240  VirginTrains    False  Tue Oct 10 15:16:08 +0000 2017   \n",
       "4    119241        105836     True  Tue Oct 10 15:17:21 +0000 2017   \n",
       "\n",
       "                                                text response_tweet_id  \\\n",
       "0  applesupport cause reply disregard tap notific...            119236   \n",
       "1  105835 business mean lot dm zip code additiona...               NaN   \n",
       "2              76328 hope change m sure will not not            119238   \n",
       "3  105836 livechat online moment contact 03331 03...            119241   \n",
       "4  virgintrain attach error message ve try leave ...            119243   \n",
       "\n",
       "   in_response_to_tweet_id  \n",
       "0                      NaN  \n",
       "1                 119239.0  \n",
       "2                      NaN  \n",
       "3                 119242.0  \n",
       "4                 119240.0  "
      ]
     },
     "execution_count": 134,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['text']=data['text'].apply(lemmatize_text)\n",
    "data.head()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.13 ('base')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "ba7d6a92d9c7295bfdcfceb88786d0d9300abf0a1802f116b1f16ab64cdf39a0"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}