code_scattertext.ipynb

{"cells":[{"attachments":{},"cell_type":"markdown","metadata":{},"source":["# <font color='darkorange'>**Term association visualization**</font>\n","\n","This notebook explains the process followed to obtain the term association graphs entailing what terms within the database appear more and less frequently in one specific category when compared to the other nine categories. The original code can be found in Jason Kessler’s work available in GitHub (<a href=\"https://github.com/JasonKessler/scattertext\">link to source</a>)."]},{"cell_type":"code","execution_count":1,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":35667,"status":"ok","timestamp":1681637124573,"user":{"displayName":"Javi AB","userId":"04948667371679011778"},"user_tz":-120},"id":"Eaxfevv7sP-j","outputId":"61f3917d-9604-4ccb-800e-90b0fc153c75"},"outputs":[{"name":"stderr","output_type":"stream","text":["2023-04-16 11:58:11.673750: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n","To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n","/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n","  from .autonotebook import tqdm as notebook_tqdm\n"]}],"source":["# Setting things up.\n","#%pip install scattertext\n","#%pip install pytextrank\n","import os\n","import pandas as pd\n","from pathlib import Path\n","import scattertext as st\n","import spacy\n","\n","os.chdir('*** Change working directory here if needed ***')\n","data_folder = Path('data/')\n","html_folder = Path('html/')"]},{"cell_type":"code","execution_count":2,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":341},"executionInfo":{"elapsed":694,"status":"error","timestamp":1681637175861,"user":{"displayName":"Javi AB","userId":"04948667371679011778"},"user_tz":-120},"id":"H_PRlbBysltV","outputId":"3d74c8a3-4862-4932-8106-bca62d2381f8"},"outputs":[{"name":"stderr","output_type":"stream","text":["/var/folders/4m/0vc6_pcd4t9bxgzklmf830080000gn/T/ipykernel_4441/3004736673.py:8: DtypeWarning: Columns (5,9) have mixed types. Specify dtype option on import or set low_memory=False.\n","  data = pd.read_csv(data_folder/'ufo_sighting_data.csv', header=0, delimiter=',', encoding='utf-8')\n"]},{"data":{"text/plain":["Date_time                                                           10/10/1949 20:30\n","city                                                                      san marcos\n","state/province                                                                    tx\n","country                                                                           us\n","UFO_shape                                                                   cylinder\n","length_of_encounter_seconds                                                     2700\n","described_duration_of_encounter                                           45 minutes\n","description                        This event took place in early fall around 194...\n","date_documented                                                            4/27/2004\n","latitude                                                                  29.8830556\n","longitude                                                                 -97.941111\n","Name: 0, dtype: object"]},"execution_count":2,"metadata":{},"output_type":"execute_result"}],"source":["# Loading alien data.\n","data = pd.read_csv(data_folder/'ufo_sighting_data.csv', header=0, delimiter=',', encoding='utf-8')\n","data.iloc[0]"]},{"cell_type":"code","execution_count":3,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":659},"executionInfo":{"elapsed":253,"status":"ok","timestamp":1681045670194,"user":{"displayName":"Katherine Williams","userId":"13170521156007312210"},"user_tz":-120},"id":"m9IM6hGCHmx9","outputId":"af35995b-9567-47b5-a92d-628af22e6158"},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>Date_time</th>\n","      <th>city</th>\n","      <th>state/province</th>\n","      <th>country</th>\n","      <th>UFO_shape</th>\n","      <th>length_of_encounter_seconds</th>\n","      <th>described_duration_of_encounter</th>\n","      <th>description</th>\n","      <th>date_documented</th>\n","      <th>latitude</th>\n","      <th>longitude</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>10/10/1949 20:30</td>\n","      <td>san marcos</td>\n","      <td>tx</td>\n","      <td>us</td>\n","      <td>cylinder</td>\n","      <td>2700</td>\n","      <td>45 minutes</td>\n","      <td>This event took place in early fall around 194...</td>\n","      <td>4/27/2004</td>\n","      <td>29.8830556</td>\n","      <td>-97.941111</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>10/10/1956 21:00</td>\n","      <td>edna</td>\n","      <td>tx</td>\n","      <td>us</td>\n","      <td>circle</td>\n","      <td>20</td>\n","      <td>1/2 hour</td>\n","      <td>My older brother and twin sister were leaving ...</td>\n","      <td>1/17/2004</td>\n","      <td>28.9783333</td>\n","      <td>-96.645833</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>10/10/1960 20:00</td>\n","      <td>kaneohe</td>\n","      <td>hi</td>\n","      <td>us</td>\n","      <td>light</td>\n","      <td>900</td>\n","      <td>15 minutes</td>\n","      <td>AS a Marine 1st Lt. flying an FJ4B fighter/att...</td>\n","      <td>1/22/2004</td>\n","      <td>21.4180556</td>\n","      <td>-157.803611</td>\n","    </tr>\n","    <tr>\n","      <th>5</th>\n","      <td>10/10/1961 19:00</td>\n","      <td>bristol</td>\n","      <td>tn</td>\n","      <td>us</td>\n","      <td>sphere</td>\n","      <td>300</td>\n","      <td>5 minutes</td>\n","      <td>My father is now 89 my brother 52 the girl wit...</td>\n","      <td>4/27/2007</td>\n","      <td>36.595</td>\n","      <td>-82.188889</td>\n","    </tr>\n","    <tr>\n","      <th>7</th>\n","      <td>10/10/1965 23:45</td>\n","      <td>norwalk</td>\n","      <td>ct</td>\n","      <td>us</td>\n","      <td>disk</td>\n","      <td>1200</td>\n","      <td>20 minutes</td>\n","      <td>A bright orange color changing to reddish colo...</td>\n","      <td>10/2/1999</td>\n","      <td>41.1175</td>\n","      <td>-73.408333</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>80327</th>\n","      <td>9/9/2013 21:15</td>\n","      <td>nashville</td>\n","      <td>tn</td>\n","      <td>us</td>\n","      <td>light</td>\n","      <td>600.0</td>\n","      <td>10 minutes</td>\n","      <td>Round from the distance/slowly changing colors...</td>\n","      <td>9/30/2013</td>\n","      <td>36.165833</td>\n","      <td>-86.784444</td>\n","    </tr>\n","    <tr>\n","      <th>80328</th>\n","      <td>9/9/2013 22:00</td>\n","      <td>boise</td>\n","      <td>id</td>\n","      <td>us</td>\n","      <td>circle</td>\n","      <td>1200.0</td>\n","      <td>20 minutes</td>\n","      <td>Boise&amp;#44 ID&amp;#44 spherical&amp;#44 20 min&amp;#44 10 r...</td>\n","      <td>9/30/2013</td>\n","      <td>43.613611</td>\n","      <td>-116.202500</td>\n","    </tr>\n","    <tr>\n","      <th>80329</th>\n","      <td>9/9/2013 22:00</td>\n","      <td>napa</td>\n","      <td>ca</td>\n","      <td>us</td>\n","      <td>other</td>\n","      <td>1200.0</td>\n","      <td>hour</td>\n","      <td>Napa UFO&amp;#44</td>\n","      <td>9/30/2013</td>\n","      <td>38.297222</td>\n","      <td>-122.284444</td>\n","    </tr>\n","    <tr>\n","      <th>80330</th>\n","      <td>9/9/2013 22:20</td>\n","      <td>vienna</td>\n","      <td>va</td>\n","      <td>us</td>\n","      <td>circle</td>\n","      <td>5.0</td>\n","      <td>5 seconds</td>\n","      <td>Saw a five gold lit cicular craft moving fastl...</td>\n","      <td>9/30/2013</td>\n","      <td>38.901111</td>\n","      <td>-77.265556</td>\n","    </tr>\n","    <tr>\n","      <th>80331</th>\n","      <td>9/9/2013 23:00</td>\n","      <td>edmond</td>\n","      <td>ok</td>\n","      <td>us</td>\n","      <td>cigar</td>\n","      <td>1020.0</td>\n","      <td>17 minutes</td>\n","      <td>2 witnesses 2  miles apart&amp;#44 Red &amp;amp; White...</td>\n","      <td>9/30/2013</td>\n","      <td>35.652778</td>\n","      <td>-97.477778</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>65114 rows × 11 columns</p>\n","</div>"],"text/plain":["              Date_time        city state/province country UFO_shape  \\\n","0      10/10/1949 20:30  san marcos             tx      us  cylinder   \n","3      10/10/1956 21:00        edna             tx      us    circle   \n","4      10/10/1960 20:00     kaneohe             hi      us     light   \n","5      10/10/1961 19:00     bristol             tn      us    sphere   \n","7      10/10/1965 23:45     norwalk             ct      us      disk   \n","...                 ...         ...            ...     ...       ...   \n","80327    9/9/2013 21:15   nashville             tn      us     light   \n","80328    9/9/2013 22:00       boise             id      us    circle   \n","80329    9/9/2013 22:00        napa             ca      us     other   \n","80330    9/9/2013 22:20      vienna             va      us    circle   \n","80331    9/9/2013 23:00      edmond             ok      us     cigar   \n","\n","      length_of_encounter_seconds described_duration_of_encounter  \\\n","0                            2700                      45 minutes   \n","3                              20                        1/2 hour   \n","4                             900                      15 minutes   \n","5                             300                       5 minutes   \n","7                            1200                      20 minutes   \n","...                           ...                             ...   \n","80327                       600.0                      10 minutes   \n","80328                      1200.0                      20 minutes   \n","80329                      1200.0                            hour   \n","80330                         5.0                       5 seconds   \n","80331                      1020.0                      17 minutes   \n","\n","                                             description date_documented  \\\n","0      This event took place in early fall around 194...       4/27/2004   \n","3      My older brother and twin sister were leaving ...       1/17/2004   \n","4      AS a Marine 1st Lt. flying an FJ4B fighter/att...       1/22/2004   \n","5      My father is now 89 my brother 52 the girl wit...       4/27/2007   \n","7      A bright orange color changing to reddish colo...       10/2/1999   \n","...                                                  ...             ...   \n","80327  Round from the distance/slowly changing colors...       9/30/2013   \n","80328  Boise&#44 ID&#44 spherical&#44 20 min&#44 10 r...       9/30/2013   \n","80329                                       Napa UFO&#44       9/30/2013   \n","80330  Saw a five gold lit cicular craft moving fastl...       9/30/2013   \n","80331  2 witnesses 2  miles apart&#44 Red &amp; White...       9/30/2013   \n","\n","         latitude   longitude  \n","0      29.8830556  -97.941111  \n","3      28.9783333  -96.645833  \n","4      21.4180556 -157.803611  \n","5          36.595  -82.188889  \n","7         41.1175  -73.408333  \n","...           ...         ...  \n","80327   36.165833  -86.784444  \n","80328   43.613611 -116.202500  \n","80329   38.297222 -122.284444  \n","80330   38.901111  -77.265556  \n","80331   35.652778  -97.477778  \n","\n","[65114 rows x 11 columns]"]},"execution_count":3,"metadata":{},"output_type":"execute_result"}],"source":["# Showing some instances of sightings in the US.\n","us_data = data[data['country'] == 'us']\n","us_data"]},{"cell_type":"code","execution_count":4,"metadata":{"id":"qRAqayaLCnli"},"outputs":[],"source":["# Initializing spacy.\n","nlp = spacy.load('en_core_web_sm')"]},{"cell_type":"code","execution_count":5,"metadata":{"id":"unSYT64Fu32u"},"outputs":[],"source":["# Creating the corpus.\n","corpus = st.CorpusFromPandas(us_data,\n","                             category_col='UFO_shape',\n","                             text_col='description',\n","                             nlp=nlp).build()"]},{"cell_type":"code","execution_count":58,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["👽 Frequent terms for chevron --> ['chevron', 'chevron shaped', 'a chevron', 'chevron shape', 'cheveron', 'large chevron', 'chevron formation', 'black chevron', 'quot;v&quot shaped', 'large boomerang']\n","👽 Frequent terms for cigar --> ['silver cigar', 'cigar shaped&#44', 'large cigar', 'cigar shaped', 'white cigar', 'cigar', 'cigar shape', 'a cigar', 'long cigar', 'orange cigar']\n","👽 Frequent terms for diamond --> ['black diamond', 'dimond', 'diamond shaped', 'diamond', 'diamond shape', 'a diamond', 'bright diamond', 'orange diamond', 'white diamond', 'diamond with']\n","👽 Frequent terms for disk --> ['silver disk', 'a disk', 'saucer with', 'saucer shaped', 'saucer', 'a saucer', 'a disc', 'disk with', 'disk shaped', 'disk shape']\n","👽 Frequent terms for formation --> ['formation of', 'formation seen', 'a formation', 'v formation', 'shaped formation', 'light formation', 'formation in', 'formation', 'of 5', 'in v']\n","👽 Frequent terms for other --> ['crescent', 'crescent shaped', 'bell', 'bell shaped', 'boomerang shaped', 'boomerang', 'of earlier', 'earlier missile', 'wing', 'result of']\n","👽 Frequent terms for oval --> ['oval craft', 'oval objects', 'oval object', 'large oval', 'orange oval', 'white oval', 'oval shaped', 'oval', 'oval shape', 'oval&#44']\n","👽 Frequent terms for rectangle --> ['rectangular craft', 'large rectangular', 'rectangle shaped', 'rectangular object', 'rectangle', 'rectangular', 'rectangular shaped', 'black rectangle', 'rectangular&#44', 'a rectangular']\n","👽 Frequent terms for triangle --> ['large triangular', 'triangle ufo', 'triangle craft', 'triangular craft', 'black triangle', 'black triangular', 'triangular ufo', 'triangular object', 'triangular shaped', 'triangle shaped']\n"]}],"source":["# Exploring the 10 most frequent words for each category of ufo.\n","cats = ['chevron', 'cigar', 'diamond', 'disk', 'formation', 'other', 'oval', 'rectangle', 'triangle']\n","\n","for cat in cats:\n","    term_freq = corpus.get_term_freq_df()\n","    term_freq[cat] = corpus.get_scaled_f_scores(cat)\n","    print('\\U0001F47D Frequent terms for', cat, '-->', list(term_freq.sort_values(by=cat, ascending=False).index[:10]))"]},{"cell_type":"code","execution_count":53,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":3389,"status":"ok","timestamp":1681047516754,"user":{"displayName":"Katherine Williams","userId":"13170521156007312210"},"user_tz":-120},"id":"X0li4R_NAFR9","outputId":"359df141-7a01-4fc4-dae2-0d2ad9fdd266"},"outputs":[{"data":{"text/plain":["8026933"]},"execution_count":53,"metadata":{},"output_type":"execute_result"}],"source":["# Creating the html file for the scatterplot. The example below works for the 'cigar' category.\n","html = st.produce_scattertext_explorer(corpus,\n","                                       category='cigar',\n","                                       category_name='for cigar',\n","                                       not_category_name='for the rest',\n","                                       width_in_pixels=1000,\n","                                       )\n","\n","open(html_folder/'ufo_cigar.html','wb').write(html.encode('utf-8'))"]}],"metadata":{"colab":{"provenance":[]},"kernelspec":{"display_name":"Python 3","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.8.1"}},"nbformat":4,"nbformat_minor":0}