update ner and ep-sentiment

worldbank · Feb 9, 2024 · 15b9ead · 15b9ead
1 parent dbef2c4
commit 15b9ead
Show file tree

Hide file tree

Showing 22 changed files with 206,677 additions and 3 deletions.
diff --git a/data/text/fiji/fiji_abc_news_ner.csv b/data/text/fiji/fiji_abc_news_ner.csv
diff --git a/data/text/fiji/fiji_rnz_news_ner.csv b/data/text/fiji/fiji_rnz_news_ner.csv
diff --git a/data/text/fiji/fiji_sun_news_ner.csv b/data/text/fiji/fiji_sun_news_ner.csv
diff --git a/data/text/marshall_islands/marshall_islands_abc_news_ner.csv b/data/text/marshall_islands/marshall_islands_abc_news_ner.csv
diff --git a/data/text/papua_new_guinea/papua_new_guinea_abc_news_ner.csv b/data/text/papua_new_guinea/papua_new_guinea_abc_news_ner.csv
diff --git a/data/text/papua_new_guinea/papua_new_guinea_rnz_news_ner.csv b/data/text/papua_new_guinea/papua_new_guinea_rnz_news_ner.csv
diff --git a/data/text/papua_new_guinea/png_business_news_ner.csv b/data/text/papua_new_guinea/png_business_news_ner.csv
diff --git a/data/text/samoa/samoa_observer_news_ner.csv b/data/text/samoa/samoa_observer_news_ner.csv
diff --git a/data/text/samoa/samoa_rnz_news_ner.csv b/data/text/samoa/samoa_rnz_news_ner.csv
diff --git a/data/text/solomon_islands/island_sun_news_ner.csv b/data/text/solomon_islands/island_sun_news_ner.csv
diff --git a/data/text/solomon_islands/sibc_news_ner.csv b/data/text/solomon_islands/sibc_news_ner.csv
diff --git a/data/text/solomon_islands/solomon_islands_abc_news_ner.csv b/data/text/solomon_islands/solomon_islands_abc_news_ner.csv
diff --git a/data/text/solomon_islands/solomon_islands_rnz_news_ner.csv b/data/text/solomon_islands/solomon_islands_rnz_news_ner.csv
diff --git a/data/text/solomon_islands/solomon_star_news_ner.csv b/data/text/solomon_islands/solomon_star_news_ner.csv
diff --git a/data/text/solomon_islands/solomon_times_news_ner.csv b/data/text/solomon_islands/solomon_times_news_ner.csv
diff --git a/data/text/tonga/matangi_news_ner.csv b/data/text/tonga/matangi_news_ner.csv
diff --git a/data/text/tonga/tonga_abc_news_ner.csv b/data/text/tonga/tonga_abc_news_ner.csv
diff --git a/data/text/vanuatu/daily_posts_news_ner.csv b/data/text/vanuatu/daily_posts_news_ner.csv
diff --git a/data/text/vanuatu/vanuatu_abc_news_ner.csv b/data/text/vanuatu/vanuatu_abc_news_ner.csv
diff --git a/docs/images/interactive/text/ep_sentiment.html b/docs/images/interactive/text/ep_sentiment.html
diff --git a/requirements.txt b/requirements.txt
@@ -1,15 +1,15 @@
 beautifulsoup4==4.12.3
 bokeh==3.2.1
 chardet==5.2.0
+numpy==1.26.0
+pandas==2.0.3
 gensim==3.8.3
 google_api_python_client==2.94.0
 lxml==4.9.3
 matplotlib==3.5.3
 networkx==3.1
 nltk==3.8.1
-pandas==2.0.3
 pdf2image==1.16.3
-Pillow==9.4.0
 Pillow==10.2.0
 pmdarima==2.0.4
 prophet==1.1.4

diff --git a/src/text/epu.py b/src/text/epu.py
@@ -57,11 +57,12 @@ class EPU:
     def __init__(self,
                  filepath: Union[str, List[str]],
                  cutoff: str,
+                 non_epu_urls: list = None,
                  econ_terms: list = ECON_LIST,
                  policy_terms: list = POLICY_LIST,
                  uncertainty_terms: list = UNCERTAINTY_LIST,
                  additional_terms: Union[List, None] = None):
-
+    
         if isinstance(filepath, str):
             self.filepath = [filepath]
         elif isinstance(filepath, list):
@@ -76,6 +77,7 @@ def __init__(self,
         self.additional_terms = additional_terms
         self.raw_files = []
         self.cutoff = cutoff
+        self.non_epu_urls = non_epu_urls if non_epu_urls is not None else []
         self.min_date = None
         self.max_date = None
         self.epu_stats = pd.DataFrame()
@@ -157,6 +159,10 @@ def get_epu_category(self, subset_condition=None):
                 raw["additional"] = raw["news"].str.lower().apply(
                     is_in_word_list, terms=self.additional_terms)
                 raw["epu"] = (raw.epu == True) & (raw.additional == True)
+
+            if raw["url"].isin(self.non_epu_urls).sum() > 0:
+                raw.loc[raw.url.isin(self.non_epu_urls), "epu"] = False
+
             self.raw_files.append((source, raw.copy()))
 
     def calculate_news_and_epu_counts(self,