Main_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "reflected-contributor",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd \n",
    "from zipfile import ZipFile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "federal-fever",
   "metadata": {},
   "outputs": [],
   "source": [
    "my_zip = ZipFile('C:/Users/ishan/OneDrive/Desktop/Hackathon File/New_dataset.zip')\n",
    "df = pd.read_csv(my_zip.open('tweet_dataset.csv'), engine = 'python')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "ancient-wedding",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['textID',\n",
       " 'sentiment',\n",
       " 'author',\n",
       " 'text',\n",
       " 'old_text',\n",
       " 'aux_id',\n",
       " 'new_sentiment',\n",
       " 'selected_text']"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "improved-advertising",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(40000, 8)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "respected-adoption",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "neutral     12668\n",
       "positive     9897\n",
       "negative     8830\n",
       "Name: new_sentiment, dtype: int64"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['new_sentiment'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "crude-victim",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>textID</th>\n",
       "      <th>sentiment</th>\n",
       "      <th>author</th>\n",
       "      <th>text</th>\n",
       "      <th>old_text</th>\n",
       "      <th>aux_id</th>\n",
       "      <th>new_sentiment</th>\n",
       "      <th>selected_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1956967341</td>\n",
       "      <td>empty</td>\n",
       "      <td>xoshayzers</td>\n",
       "      <td>i know  i was listenin to bad habit earlier a...</td>\n",
       "      <td>@tiffanylue i know  i was listenin to bad habi...</td>\n",
       "      <td>p1000000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1956967666</td>\n",
       "      <td>sadness</td>\n",
       "      <td>wannamama</td>\n",
       "      <td>Layin n bed with a headache  ughhhh...waitin o...</td>\n",
       "      <td>Layin n bed with a headache  ughhhh...waitin o...</td>\n",
       "      <td>c811396dc2</td>\n",
       "      <td>negative</td>\n",
       "      <td>headache</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1956967696</td>\n",
       "      <td>sadness</td>\n",
       "      <td>coolfunky</td>\n",
       "      <td>Funeral ceremony...gloomy friday...</td>\n",
       "      <td>Funeral ceremony...gloomy friday...</td>\n",
       "      <td>9063631ab1</td>\n",
       "      <td>negative</td>\n",
       "      <td>gloomy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1956967789</td>\n",
       "      <td>enthusiasm</td>\n",
       "      <td>czareaquino</td>\n",
       "      <td>wants to hang out with friends SOON!</td>\n",
       "      <td>wants to hang out with friends SOON!</td>\n",
       "      <td>2a815f151d</td>\n",
       "      <td>positive</td>\n",
       "      <td>wants to hang out with friends SOON!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1956968416</td>\n",
       "      <td>neutral</td>\n",
       "      <td>xkilljoyx</td>\n",
       "      <td>We want to trade with someone who has Houston...</td>\n",
       "      <td>@dannycastillo We want to trade with someone w...</td>\n",
       "      <td>82565a56d3</td>\n",
       "      <td>neutral</td>\n",
       "      <td>We want to trade with someone who has Houston ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       textID   sentiment       author  \\\n",
       "0  1956967341       empty   xoshayzers   \n",
       "1  1956967666     sadness    wannamama   \n",
       "2  1956967696     sadness    coolfunky   \n",
       "3  1956967789  enthusiasm  czareaquino   \n",
       "4  1956968416     neutral    xkilljoyx   \n",
       "\n",
       "                                                text  \\\n",
       "0   i know  i was listenin to bad habit earlier a...   \n",
       "1  Layin n bed with a headache  ughhhh...waitin o...   \n",
       "2                Funeral ceremony...gloomy friday...   \n",
       "3               wants to hang out with friends SOON!   \n",
       "4   We want to trade with someone who has Houston...   \n",
       "\n",
       "                                            old_text       aux_id  \\\n",
       "0  @tiffanylue i know  i was listenin to bad habi...  p1000000000   \n",
       "1  Layin n bed with a headache  ughhhh...waitin o...   c811396dc2   \n",
       "2                Funeral ceremony...gloomy friday...   9063631ab1   \n",
       "3               wants to hang out with friends SOON!   2a815f151d   \n",
       "4  @dannycastillo We want to trade with someone w...   82565a56d3   \n",
       "\n",
       "  new_sentiment                                      selected_text  \n",
       "0           NaN                                                NaN  \n",
       "1      negative                                           headache  \n",
       "2      negative                                             gloomy  \n",
       "3      positive               wants to hang out with friends SOON!  \n",
       "4       neutral  We want to trade with someone who has Houston ...  "
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "complete-public",
   "metadata": {},
   "outputs": [],
   "source": [
    "options = ['positive','negative']\n",
    "df = df[df['new_sentiment'].isin(options)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "synthetic-softball",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "positive    9897\n",
       "negative    8830\n",
       "Name: new_sentiment, dtype: int64"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['new_sentiment'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "indoor-kitty",
   "metadata": {},
   "outputs": [],
   "source": [
    "features = df['text']\n",
    "labels = df['new_sentiment']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "polar-italian",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "features_train,features_test,labels_train,labels_test = train_test_split(features,labels,test_size=0.2,random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "transparent-italy",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "Vect = TfidfVectorizer(min_df=5).fit(features_train)\n",
    "\n",
    "features_train_vectorized = Vect.transform(features_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "arctic-psychiatry",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "model = LogisticRegression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "architectural-irrigation",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression()"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(features_train_vectorized,labels_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "declared-pricing",
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = model.predict(Vect.transform(features_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "exclusive-railway",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "cm = confusion_matrix(labels_test,predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "civic-pizza",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8857447944474106"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "accuracy_score(labels_test,predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "facial-layer",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "file = open(\"pickle_model.pkl\",\"wb\")\n",
    "pickle.dump(model,file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "acceptable-bankruptcy",
   "metadata": {},
   "outputs": [],
   "source": [
    "file_vocab = open(\"features.pkl\",\"wb\")\n",
    "pickle.dump(Vect.vocabulary_,file_vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "monetary-excuse",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}