Merge pull request #938 from GbCderr14/main

Fixed scraping function of fetching research topics from academia.
Clueless-Community · May 14, 2024 · 0c09dc8 · 0c09dc8
2 parents f336697 + 9b006d9
commit 0c09dc8
Show file tree

Hide file tree

Showing 4 changed files with 5 additions and 6 deletions.
diff --git a/dev-documentation.md b/dev-documentation.md
@@ -1471,7 +1471,7 @@ academia = Academia()
 
 | Method                        | Details                                                               |
 | ----------------------------- | --------------------------------------------------------------------- |
-| `get_research_topics(letter)` | Fetches and returns research topics starting with the given letter.   |
+| `get_research_topics()`       | Fetches and returns research topics.                                  |
 | `get_research_papers(search)` | Fetches and returns research papers related to the given search term. |
 
 ---

diff --git a/documentation.md b/documentation.md
@@ -344,7 +344,7 @@ academia = academia.Academia()
 
 | Method                        | Details                                                               |
 | ----------------------------- | --------------------------------------------------------------------- |
-| `get_research_topics(letter)` | Fetches and returns research topics starting with the given letter.   |
+| `get_research_topics()`       | Fetches and returns research topics.                                  |
 | `get_research_papers(search)` | Fetches and returns research papers related to the given search term. |
 
 ---

diff --git a/src/scrape_up/academia/academia.py b/src/scrape_up/academia/academia.py
@@ -22,7 +22,7 @@ def __init__(self):
             "User-Agent": "Mozilla/5.0 (Windows NT 6.3; Win 64 ; x64) Apple WeKit /537.36(KHTML , like Gecko) Chrome/80.0.3987.162 Safari/537.36"
         }
 
-    def get_research_topics(self, topic="None"):
+    def get_research_topics(self):
         """
         Fetches and returns research topics starting with the given letter.\n
         Param `letter`: The letter to filter research topics (default is "None" to get all topics).\n
@@ -45,8 +45,7 @@ def get_research_topics(self, topic="None"):
         ```
         """
         try:
-            letter = topic.capitalize()
-            url = f"https://www.academia.edu/topics/{letter}"
+            url = f"https://www.academia.edu/topics/"
             html_text = requests.get(url, headers=self.headers).text
             soup = BeautifulSoup(html_text, "lxml")
 

diff --git a/src/test/academia_test.py b/src/test/academia_test.py
@@ -8,7 +8,7 @@ def setUp(self):
 
     def test_get_research_topics(self):
         academia = Academia()
-        result = academia.get_research_topics(topic="Machine Learning")
+        result = academia.get_research_topics()
         self.assertIsNotNone(result)
         self.assertIsInstance(result, list)