diff --git a/backend/howtheyvote/analysis/votes.py b/backend/howtheyvote/analysis/votes.py index 38eb3ce47..3b7a48819 100644 --- a/backend/howtheyvote/analysis/votes.py +++ b/backend/howtheyvote/analysis/votes.py @@ -56,9 +56,9 @@ def _make_group_key(self, date: datetime.date, vote: Vote) -> str | None: class MainVoteAnalyzer: - """This analyzer checks the vote description for common keywords that indicate - that the vote is a main vote in its vote group. Only main votes are displayed - in index pages and are searchable.""" + """This analyzer checks the vote description and title for common keywords + that indicate that the vote is a main vote in its vote group. Only main + votes are displayed in index pages and are searchable.""" MAIN_DESCRIPTIONS = set( [ @@ -94,27 +94,52 @@ class MainVoteAnalyzer: ] ) - def __init__(self, vote_id: int, description: str | None): + MAIN_TITLES = set( + [ + "election de la commission", + "election of the commission", + ] + ) + + def __init__(self, vote_id: int, description: str | None, title: str | None): self.vote_id = vote_id self.description = description + self.title = title def run(self) -> Fragment | None: + if self._description_is_main() or self._title_is_main(): + return Fragment( + model="Vote", + source_id=self.vote_id, + source_name=type(self).__name__, + group_key=self.vote_id, + data={"is_main": True}, + ) + + return None + + def _description_is_main(self) -> bool: if not self.description: - return None + return False description = unidecode(self.description).lower() parts = description.split(" - ") - if all([part not in self.MAIN_DESCRIPTIONS for part in parts]): - return None + if any([part in self.MAIN_DESCRIPTIONS for part in parts]): + return True - return Fragment( - model="Vote", - source_id=self.vote_id, - source_name=type(self).__name__, - group_key=self.vote_id, - data={"is_main": True}, - ) + return False + + def _title_is_main(self) -> bool: + if not self.title: + return False + + title = unidecode(self.title).lower() + + if title in self.MAIN_TITLES: + return True + + return False class FeaturedVotesAnalyzer: diff --git a/backend/howtheyvote/pipelines/rcv_list.py b/backend/howtheyvote/pipelines/rcv_list.py index b463d4c05..9c11ffcff 100644 --- a/backend/howtheyvote/pipelines/rcv_list.py +++ b/backend/howtheyvote/pipelines/rcv_list.py @@ -234,7 +234,9 @@ def _analyze_main_votes(self) -> None: writer = BulkWriter() for vote in self._votes(): - analyzer = MainVoteAnalyzer(vote.id, vote.description) + analyzer = MainVoteAnalyzer( + vote_id=vote.id, description=vote.description, title=vote.title + ) writer.add(analyzer.run()) writer.flush() diff --git a/backend/tests/__init__.py b/backend/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/tests/analysis/test_votes.py b/backend/tests/analysis/test_votes.py new file mode 100644 index 000000000..3f53e8bcd --- /dev/null +++ b/backend/tests/analysis/test_votes.py @@ -0,0 +1,64 @@ +from howtheyvote.analysis.votes import MainVoteAnalyzer +from howtheyvote.models.common import Fragment + +from ..helpers import record_to_dict + + +def test_main_vote_analyzer_description(): + analyzer = MainVoteAnalyzer( + vote_id=1, + description="Am 123", + title="Lorem Ipsum", + ) + assert analyzer.run() is None + + analyzer = MainVoteAnalyzer( + vote_id=2, + description="Proposition de résolution", + title="Lorem ipsum", + ) + expected = Fragment( + model="Vote", + source_id=2, + source_name="MainVoteAnalyzer", + group_key=2, + data={"is_main": True}, + ) + assert record_to_dict(analyzer.run()) == record_to_dict(expected) + + analyzer = MainVoteAnalyzer( + vote_id=3, + description="Accord provisoire - Am 123", + title="Lorem ipsum", + ) + expected = Fragment( + model="Vote", + source_id=3, + source_name="MainVoteAnalyzer", + group_key=3, + data={"is_main": True}, + ) + assert record_to_dict(analyzer.run()) == record_to_dict(expected) + + +def test_main_vote_analyzer_title(): + analyzer = MainVoteAnalyzer( + vote_id=1, + description=None, + title="Ordre du jour de mardi", + ) + assert analyzer.run() is None + + expected = Fragment( + model="Vote", + source_id=2, + source_name="MainVoteAnalyzer", + group_key=2, + data={"is_main": True}, + ) + analyzer = MainVoteAnalyzer( + vote_id=2, + description=None, + title="Élection de la Commission", + ) + assert record_to_dict(analyzer.run()) == record_to_dict(expected) diff --git a/backend/tests/helpers.py b/backend/tests/helpers.py new file mode 100644 index 000000000..b25e3abb3 --- /dev/null +++ b/backend/tests/helpers.py @@ -0,0 +1,7 @@ +from typing import Any + +from sqlalchemy.orm import DeclarativeBase + + +def record_to_dict(record: DeclarativeBase) -> dict[str, Any]: + return {c.name: getattr(record, c.name) for c in record.__table__.columns} diff --git a/backend/tests/scrapers/helpers.py b/backend/tests/scrapers/helpers.py index ed3099538..c8be91285 100644 --- a/backend/tests/scrapers/helpers.py +++ b/backend/tests/scrapers/helpers.py @@ -1,14 +1,7 @@ from pathlib import Path -from typing import Any - -from sqlalchemy.orm import DeclarativeBase FIXTURES_BASE = Path(__file__).resolve().parent / "data" def load_fixture(path: str) -> str: return FIXTURES_BASE.joinpath(path).read_text() - - -def record_to_dict(record: DeclarativeBase) -> dict[str, Any]: - return {c.name: getattr(record, c.name) for c in record.__table__.columns} diff --git a/backend/tests/scrapers/test_votes.py b/backend/tests/scrapers/test_votes.py index d3c953faf..49c137933 100644 --- a/backend/tests/scrapers/test_votes.py +++ b/backend/tests/scrapers/test_votes.py @@ -11,7 +11,8 @@ RCVListScraper, ) -from .helpers import load_fixture, record_to_dict +from ..helpers import record_to_dict +from .helpers import load_fixture def test_rcv_list_scraper(responses):