Fix parsing (#6)

* Download test files with a script

* Fix frontpage parsing

* Fix article comment parsing

* Add mocks

pyproject.toml

@@ -17,9 +17,9 @@ classifiers=[
 ]
 
 [tool.poetry.dependencies]
-python = ">=3.6"
+python = ">=3.7"
 requests-html = "^0.10.0"
-dateparser = "^0.7.0"
+dateparser = "1.1.8"
 tenacity = "^6.2.0"
 
 [tool.poetry.dev-dependencies]

scripts/download_test_files.py

@@ -0,0 +1,23 @@
+from pathlib import Path
+
+from tweakers.utils import get
+
+URLS = {
+    "topic": "https://gathering.tweakers.net/forum/list_messages/1908208",
+    "frontpage": "https://tweakers.net/",
+    "article": "https://tweakers.net/nieuws/212172/whatsapp-introduceert-functie-voor-korte-videoberichten.html",
+    "user": "https://tweakers.net/gallery/1/",
+    "active_topics": "https://gathering.tweakers.net/forum/list_activetopics",
+    "find": "https://tweakers.net/forum/find?keyword=playstation",
+}
+
+
+for name, url in URLS.items():
+    print("Downloading", name)
+    html = get(url).text
+    if "Sorry, je gaat even iets te snel" in html:
+        raise Exception("Still rate limited???")
+    path = Path(f"tests/pages/{name}.html")
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("w") as f:
+        f.write(html)

tests/mocks.py

@@ -0,0 +1,25 @@
+import pytest
+from requests_html import HTML, HTMLResponse, HTMLSession
+
+
+def mock_get(url: str):
+    if "/nieuws/" in url:
+        case = "article"
+    elif "/list_messages/" in url:
+        case = "topic"
+    elif "/gallery/" in url:
+        case = "user"
+    elif "/list_activetopics" in url:
+        case = "active_topics"
+    elif "forum/find" in url:
+        case = "find"
+    elif url in ("https://tweakers.net/", "https://tweakers.net"):
+        case = "frontpage"
+    else:
+        raise NotImplementedError(f"Url {url} not implemented")
+
+    with open(f"tests/pages/{case}.html") as f:
+        response = HTMLResponse(session=HTMLSession)
+        response._html = HTML(html=f.read())
+    return response
+

tests/test_frontpage.py

@@ -1,11 +1,15 @@
+from unittest import mock
+from tests.mocks import mock_get
 from tweakers import frontpage
 
 
+@mock.patch("tweakers.frontpage.get", mock_get)
 def test_articles():
     articles = frontpage.articles()
     assert len(articles) > 0
 
 
+@mock.patch("tweakers.frontpage.get", mock_get)
 def test_article_comments():
     article = frontpage.Article(
         url="https://tweakers.net/nieuws/148534/amd-maakt-meer-winst-dankzij-goede-verkopen-van-ryzen-cpus.html"

tests/test_gathering.py

@@ -1,9 +1,13 @@
+from unittest import mock
+from tests.mocks import mock_get
 from tweakers import gathering
 
 
+@mock.patch("tweakers.gathering.get", mock_get)
 def test_active_topics():
     assert len(gathering.active_topics()) > 0
 
 
+@mock.patch("tweakers.gathering.get", mock_get)
 def test_search():
     assert len(gathering.search("tweakers")) > 0

tweakers/parsers.py

@@ -84,12 +84,12 @@ def topic_comments(html: Union[HTML, str]) -> Generator[dict, None, None]:
 
 
 def frontpage_articles(html: HTML) -> Generator[dict, None, None]:
-    for tr in html.find("tr.headline.news"):
+    for tr in html.find(".headline"):
         topic: Dict = {
-            "title": tr.find(".title a", first=True).text,
-            "url": tr.find(".title a", first=True).attrs["href"],
+            "title": tr.text.strip(),
+            "url": tr.find("a.headline--anchor", first=True).attrs["href"],
             "comment_count": get_comment_count(tr),
-            "publication_time": tr.find(".publicationTime", first=True).text,
+            "publication_time": tr.find(".headline--time", first=True).text,
         }
         yield topic
 
@@ -102,7 +102,7 @@ def article_comments(html: HTML) -> Generator[dict, None, None]:
                 _get_text(div, selector="a.date"), languages=["nl"]
             ),
             "text": _get_text(div, selector=".reactieContent"),
-            "score": int(_get_text(div, selector="a.scoreButton")),
+            "score": _get_text(div, selector="a.scoreButton"),
         }
         yield comment