B-sentiment-analysis-1.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"provenance":[{"file_id":"1BHkP5HWN9wwJmr_vQNqzRxHmVJn6ufql","timestamp":1678699083605}],"authorship_tag":"ABX9TyN3V5U3K+OCZwc87VJwuNa1"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"WB9ZFSI6VFfe","executionInfo":{"status":"ok","timestamp":1680613314655,"user_tz":-330,"elapsed":28172,"user":{"displayName":"Sudeshna Das","userId":"08332010396883382094"}},"outputId":"eb551e54-1a15-45ce-9316-da8d6be51497"},"outputs":[{"output_type":"stream","name":"stdout","text":["Mounted at /content/drive/\n"]}],"source":["# mount drive to access data\n","from google.colab import drive\n","drive.mount('/content/drive/')"]},{"cell_type":"code","source":["# change path HERE to reflect your folder structure\n","my_path = '/content/drive/MyDrive/suicide-project/data/'"],"metadata":{"id":"ZmcUvyv8oY0I"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# create dataframe from csv data file\n","\n","import pandas as pd\n","from ast import literal_eval\n","df = pd.read_csv(my_path + 'PROCESSED_TEXT_CORPUS.csv', usecols = ['date','post_id','without_stopwords_body'],\n","                 converters={'without_stopwords_body': literal_eval})\n","df.info()"],"metadata":{"id":"AWerUJrKWLde","executionInfo":{"status":"ok","timestamp":1680613462253,"user_tz":-330,"elapsed":146217,"user":{"displayName":"Sudeshna Das","userId":"08332010396883382094"}},"colab":{"base_uri":"https://localhost:8080/"},"outputId":"5ecb773b-65d7-4bbd-f06e-aee9156a00fd"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["<class 'pandas.core.frame.DataFrame'>\n","RangeIndex: 1337982 entries, 0 to 1337981\n","Data columns (total 3 columns):\n"," #   Column                  Non-Null Count    Dtype \n","---  ------                  --------------    ----- \n"," 0   post_id                 1337982 non-null  int64 \n"," 1   date                    1337982 non-null  object\n"," 2   without_stopwords_body  1337982 non-null  object\n","dtypes: int64(1), object(2)\n","memory usage: 30.6+ MB\n"]}]},{"cell_type":"code","source":["df.head(2)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":112},"id":"cFXRcC5VYkwe","executionInfo":{"status":"ok","timestamp":1680613462255,"user_tz":-330,"elapsed":30,"user":{"displayName":"Sudeshna Das","userId":"08332010396883382094"}},"outputId":"c871eb62-3b9a-457d-c463-4a594dc33e4e"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["   post_id         date        without_stopwords_body\n","0  1742806  Oct 6, 2022            [one, nice, thing]\n","1  1742807  Oct 6, 2022  [care, well, beings, others]"],"text/html":["\n","  <div id=\"df-2d86a708-ebb8-440d-b5e4-9db0972c08cd\">\n","    <div class=\"colab-df-container\">\n","      <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>post_id</th>\n","      <th>date</th>\n","      <th>without_stopwords_body</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>1742806</td>\n","      <td>Oct 6, 2022</td>\n","      <td>[one, nice, thing]</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>1742807</td>\n","      <td>Oct 6, 2022</td>\n","      <td>[care, well, beings, others]</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>\n","      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-2d86a708-ebb8-440d-b5e4-9db0972c08cd')\"\n","              title=\"Convert this dataframe to an interactive table.\"\n","              style=\"display:none;\">\n","        \n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","       width=\"24px\">\n","    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n","    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n","  </svg>\n","      </button>\n","      \n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      flex-wrap:wrap;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","      <script>\n","        const buttonEl =\n","          document.querySelector('#df-2d86a708-ebb8-440d-b5e4-9db0972c08cd button.colab-df-convert');\n","        buttonEl.style.display =\n","          google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","        async function convertToInteractive(key) {\n","          const element = document.querySelector('#df-2d86a708-ebb8-440d-b5e4-9db0972c08cd');\n","          const dataTable =\n","            await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                     [key], {});\n","          if (!dataTable) return;\n","\n","          const docLinkHtml = 'Like what you see? Visit the ' +\n","            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","            + ' to learn more about interactive tables.';\n","          element.innerHTML = '';\n","          dataTable['output_type'] = 'display_data';\n","          await google.colab.output.renderOutput(dataTable, element);\n","          const docLink = document.createElement('div');\n","          docLink.innerHTML = docLinkHtml;\n","          element.appendChild(docLink);\n","        }\n","      </script>\n","    </div>\n","  </div>\n","  "]},"metadata":{},"execution_count":5}]},{"cell_type":"code","source":["# lemmatization using nltk/wordnet\n","import nltk\n","nltk.download('wordnet')\n","nltk.download('omw-1.4')\n","from nltk.stem import WordNetLemmatizer\n","wordnet_lem = WordNetLemmatizer()\n","df['lemmatized_body'] = df['without_stopwords_body'].apply(lambda x: ' '.join([wordnet_lem.lemmatize(word) for word in x]))\n","df.head(2)"],"metadata":{"id":"bRzb6QSwFBai","executionInfo":{"status":"ok","timestamp":1680613659547,"user_tz":-330,"elapsed":197311,"user":{"displayName":"Sudeshna Das","userId":"08332010396883382094"}},"colab":{"base_uri":"https://localhost:8080/","height":147},"outputId":"71543be9-7753-4e3f-dd8d-cf9400cb59b5"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stderr","text":["[nltk_data] Downloading package wordnet to /root/nltk_data...\n","[nltk_data] Downloading package omw-1.4 to /root/nltk_data...\n"]},{"output_type":"execute_result","data":{"text/plain":["   post_id         date        without_stopwords_body         lemmatized_body\n","0  1742806  Oct 6, 2022            [one, nice, thing]          one nice thing\n","1  1742807  Oct 6, 2022  [care, well, beings, others]  care well being others"],"text/html":["\n","  <div id=\"df-e6278443-a582-494c-933a-85f0b8a79d6c\">\n","    <div class=\"colab-df-container\">\n","      <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>post_id</th>\n","      <th>date</th>\n","      <th>without_stopwords_body</th>\n","      <th>lemmatized_body</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>1742806</td>\n","      <td>Oct 6, 2022</td>\n","      <td>[one, nice, thing]</td>\n","      <td>one nice thing</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>1742807</td>\n","      <td>Oct 6, 2022</td>\n","      <td>[care, well, beings, others]</td>\n","      <td>care well being others</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>\n","      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e6278443-a582-494c-933a-85f0b8a79d6c')\"\n","              title=\"Convert this dataframe to an interactive table.\"\n","              style=\"display:none;\">\n","        \n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","       width=\"24px\">\n","    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n","    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n","  </svg>\n","      </button>\n","      \n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      flex-wrap:wrap;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","      <script>\n","        const buttonEl =\n","          document.querySelector('#df-e6278443-a582-494c-933a-85f0b8a79d6c button.colab-df-convert');\n","        buttonEl.style.display =\n","          google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","        async function convertToInteractive(key) {\n","          const element = document.querySelector('#df-e6278443-a582-494c-933a-85f0b8a79d6c');\n","          const dataTable =\n","            await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                     [key], {});\n","          if (!dataTable) return;\n","\n","          const docLinkHtml = 'Like what you see? Visit the ' +\n","            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","            + ' to learn more about interactive tables.';\n","          element.innerHTML = '';\n","          dataTable['output_type'] = 'display_data';\n","          await google.colab.output.renderOutput(dataTable, element);\n","          const docLink = document.createElement('div');\n","          docLink.innerHTML = docLinkHtml;\n","          element.appendChild(docLink);\n","        }\n","      </script>\n","    </div>\n","  </div>\n","  "]},"metadata":{},"execution_count":6}]},{"cell_type":"code","source":["# sentiment analysis using vader-lexicon (nltk)\n","import nltk\n","nltk.download('vader_lexicon')\n","from nltk.sentiment import SentimentIntensityAnalyzer\n","analyzer = SentimentIntensityAnalyzer()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"MnaUSAeDPd9n","executionInfo":{"status":"ok","timestamp":1680613659548,"user_tz":-330,"elapsed":27,"user":{"displayName":"Sudeshna Das","userId":"08332010396883382094"}},"outputId":"86e0b186-90e0-4ca6-8af9-855b8ea96076"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stderr","text":["[nltk_data] Downloading package vader_lexicon to /root/nltk_data...\n"]}]},{"cell_type":"code","source":["df['polarity'] = df['lemmatized_body'].apply(lambda x: analyzer.polarity_scores(x))"],"metadata":{"id":"pO4e0jW9h-af"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["df = pd.concat([df.drop(['polarity'], axis=1), df['polarity'].apply(pd.Series)], axis=1)\n","df['sentiment'] = df['compound'].apply(lambda x: 'positive' if x > 0 else 'neutral' if x == 0 else 'negative')\n","df.head(3)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":143},"id":"i1wYkkkvj11e","executionInfo":{"status":"ok","timestamp":1680614806479,"user_tz":-330,"elapsed":460812,"user":{"displayName":"Sudeshna Das","userId":"08332010396883382094"}},"outputId":"15a287d4-0cc8-402a-ee49-96955a292869"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["   post_id         date        without_stopwords_body         lemmatized_body  \\\n","0  1742806  Oct 6, 2022            [one, nice, thing]          one nice thing   \n","1  1742807  Oct 6, 2022  [care, well, beings, others]  care well being others   \n","2  1742808  Oct 6, 2022               [arent, karens]             arent karen   \n","\n","   neg    neu    pos  compound sentiment  \n","0  0.0  0.417  0.583    0.4215  positive  \n","1  0.0  0.274  0.726    0.6486  positive  \n","2  0.0  1.000  0.000    0.0000   neutral  "],"text/html":["\n","  <div id=\"df-9c8b1cd8-3bef-41fb-ae84-d1a59edc65fa\">\n","    <div class=\"colab-df-container\">\n","      <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>post_id</th>\n","      <th>date</th>\n","      <th>without_stopwords_body</th>\n","      <th>lemmatized_body</th>\n","      <th>neg</th>\n","      <th>neu</th>\n","      <th>pos</th>\n","      <th>compound</th>\n","      <th>sentiment</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>1742806</td>\n","      <td>Oct 6, 2022</td>\n","      <td>[one, nice, thing]</td>\n","      <td>one nice thing</td>\n","      <td>0.0</td>\n","      <td>0.417</td>\n","      <td>0.583</td>\n","      <td>0.4215</td>\n","      <td>positive</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>1742807</td>\n","      <td>Oct 6, 2022</td>\n","      <td>[care, well, beings, others]</td>\n","      <td>care well being others</td>\n","      <td>0.0</td>\n","      <td>0.274</td>\n","      <td>0.726</td>\n","      <td>0.6486</td>\n","      <td>positive</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>1742808</td>\n","      <td>Oct 6, 2022</td>\n","      <td>[arent, karens]</td>\n","      <td>arent karen</td>\n","      <td>0.0</td>\n","      <td>1.000</td>\n","      <td>0.000</td>\n","      <td>0.0000</td>\n","      <td>neutral</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>\n","      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-9c8b1cd8-3bef-41fb-ae84-d1a59edc65fa')\"\n","              title=\"Convert this dataframe to an interactive table.\"\n","              style=\"display:none;\">\n","        \n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","       width=\"24px\">\n","    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n","    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n","  </svg>\n","      </button>\n","      \n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      flex-wrap:wrap;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","      <script>\n","        const buttonEl =\n","          document.querySelector('#df-9c8b1cd8-3bef-41fb-ae84-d1a59edc65fa button.colab-df-convert');\n","        buttonEl.style.display =\n","          google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","        async function convertToInteractive(key) {\n","          const element = document.querySelector('#df-9c8b1cd8-3bef-41fb-ae84-d1a59edc65fa');\n","          const dataTable =\n","            await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                     [key], {});\n","          if (!dataTable) return;\n","\n","          const docLinkHtml = 'Like what you see? Visit the ' +\n","            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","            + ' to learn more about interactive tables.';\n","          element.innerHTML = '';\n","          dataTable['output_type'] = 'display_data';\n","          await google.colab.output.renderOutput(dataTable, element);\n","          const docLink = document.createElement('div');\n","          docLink.innerHTML = docLinkHtml;\n","          element.appendChild(docLink);\n","        }\n","      </script>\n","    </div>\n","  </div>\n","  "]},"metadata":{},"execution_count":9}]},{"cell_type":"code","source":["df.drop(['without_stopwords_body','pos','neu','neg'], axis=1, inplace=True)\n","df.info()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"bqKqwdtEqRKN","executionInfo":{"status":"ok","timestamp":1680614806480,"user_tz":-330,"elapsed":46,"user":{"displayName":"Sudeshna Das","userId":"08332010396883382094"}},"outputId":"83aaaa53-d837-4472-9194-2e4338860a74"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["<class 'pandas.core.frame.DataFrame'>\n","RangeIndex: 1337982 entries, 0 to 1337981\n","Data columns (total 5 columns):\n"," #   Column           Non-Null Count    Dtype  \n","---  ------           --------------    -----  \n"," 0   post_id          1337982 non-null  int64  \n"," 1   date             1337982 non-null  object \n"," 2   lemmatized_body  1337982 non-null  object \n"," 3   compound         1337982 non-null  float64\n"," 4   sentiment        1337982 non-null  object \n","dtypes: float64(1), int64(1), object(3)\n","memory usage: 51.0+ MB\n"]}]},{"cell_type":"code","source":["df.to_csv(my_path+'sentiment.csv', sep=',', header=True, index=False)"],"metadata":{"id":"ZHE3xEkgxxkW"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["df['date_n'] = pd.to_datetime(df['date'])    # converting string type to datetime object\n","list_year = [i.split(\" \")[0][0:4] for i in list(df['date_n'].astype(str))]\n","list_month = [i.split(\" \")[0][5:7] for i in list(df['date_n'].astype(str))]\n","df['year'] = list_year\n","df['month'] = list_month\n","df.drop(['date_n'], axis=1, inplace=True)"],"metadata":{"id":"dhNiszoUc0NQ"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["df.info()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"2uFdI17DdEJb","executionInfo":{"status":"ok","timestamp":1680614908126,"user_tz":-330,"elapsed":2575,"user":{"displayName":"Sudeshna Das","userId":"08332010396883382094"}},"outputId":"5ad47f39-3681-4d58-8fe0-f12bb8544126"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["<class 'pandas.core.frame.DataFrame'>\n","RangeIndex: 1337982 entries, 0 to 1337981\n","Data columns (total 7 columns):\n"," #   Column           Non-Null Count    Dtype  \n","---  ------           --------------    -----  \n"," 0   post_id          1337982 non-null  int64  \n"," 1   date             1337982 non-null  object \n"," 2   lemmatized_body  1337982 non-null  object \n"," 3   compound         1337982 non-null  float64\n"," 4   sentiment        1337982 non-null  object \n"," 5   year             1337982 non-null  object \n"," 6   month            1337982 non-null  object \n","dtypes: float64(1), int64(1), object(5)\n","memory usage: 71.5+ MB\n"]}]},{"cell_type":"code","source":["df.drop(['post_id','date','lemmatized_body','compound'], axis=1, inplace=True)"],"metadata":{"id":"tdBj58ccemHo"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["dfcounts = df.groupby(['year','month', 'sentiment']).size().reset_index()\n","dfcounts"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":424},"id":"e6an-tw3gtcX","executionInfo":{"status":"ok","timestamp":1680616354752,"user_tz":-330,"elapsed":1794,"user":{"displayName":"Sudeshna Das","userId":"08332010396883382094"}},"outputId":"cf498965-e083-4edd-b512-96dcc9917386"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["     year month sentiment      0\n","0    2018    03  negative    106\n","1    2018    03   neutral     20\n","2    2018    03  positive    117\n","3    2018    04  negative    654\n","4    2018    04   neutral    164\n","..    ...   ...       ...    ...\n","163  2022    09   neutral   3759\n","164  2022    09  positive  13005\n","165  2022    10  negative   2313\n","166  2022    10   neutral    762\n","167  2022    10  positive   2727\n","\n","[168 rows x 4 columns]"],"text/html":["\n","  <div id=\"df-1ce8a756-3828-46a9-9bd3-96ff9de516f0\">\n","    <div class=\"colab-df-container\">\n","      <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>year</th>\n","      <th>month</th>\n","      <th>sentiment</th>\n","      <th>0</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>2018</td>\n","      <td>03</td>\n","      <td>negative</td>\n","      <td>106</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>2018</td>\n","      <td>03</td>\n","      <td>neutral</td>\n","      <td>20</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>2018</td>\n","      <td>03</td>\n","      <td>positive</td>\n","      <td>117</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>2018</td>\n","      <td>04</td>\n","      <td>negative</td>\n","      <td>654</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>2018</td>\n","      <td>04</td>\n","      <td>neutral</td>\n","      <td>164</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>163</th>\n","      <td>2022</td>\n","      <td>09</td>\n","      <td>neutral</td>\n","      <td>3759</td>\n","    </tr>\n","    <tr>\n","      <th>164</th>\n","      <td>2022</td>\n","      <td>09</td>\n","      <td>positive</td>\n","      <td>13005</td>\n","    </tr>\n","    <tr>\n","      <th>165</th>\n","      <td>2022</td>\n","      <td>10</td>\n","      <td>negative</td>\n","      <td>2313</td>\n","    </tr>\n","    <tr>\n","      <th>166</th>\n","      <td>2022</td>\n","      <td>10</td>\n","      <td>neutral</td>\n","      <td>762</td>\n","    </tr>\n","    <tr>\n","      <th>167</th>\n","      <td>2022</td>\n","      <td>10</td>\n","      <td>positive</td>\n","      <td>2727</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>168 rows × 4 columns</p>\n","</div>\n","      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-1ce8a756-3828-46a9-9bd3-96ff9de516f0')\"\n","              title=\"Convert this dataframe to an interactive table.\"\n","              style=\"display:none;\">\n","        \n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","       width=\"24px\">\n","    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n","    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n","  </svg>\n","      </button>\n","      \n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      flex-wrap:wrap;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","      <script>\n","        const buttonEl =\n","          document.querySelector('#df-1ce8a756-3828-46a9-9bd3-96ff9de516f0 button.colab-df-convert');\n","        buttonEl.style.display =\n","          google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","        async function convertToInteractive(key) {\n","          const element = document.querySelector('#df-1ce8a756-3828-46a9-9bd3-96ff9de516f0');\n","          const dataTable =\n","            await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                     [key], {});\n","          if (!dataTable) return;\n","\n","          const docLinkHtml = 'Like what you see? Visit the ' +\n","            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","            + ' to learn more about interactive tables.';\n","          element.innerHTML = '';\n","          dataTable['output_type'] = 'display_data';\n","          await google.colab.output.renderOutput(dataTable, element);\n","          const docLink = document.createElement('div');\n","          docLink.innerHTML = docLinkHtml;\n","          element.appendChild(docLink);\n","        }\n","      </script>\n","    </div>\n","  </div>\n","  "]},"metadata":{},"execution_count":22}]},{"cell_type":"code","source":["dfcounts.to_csv(my_path+'sentiment_all_month_year.csv', sep=',', header=True, index=False)"],"metadata":{"id":"JTFbrUVwip_3"},"execution_count":null,"outputs":[]}]}