diff --git a/city_scrapers/spiders/chi_municipal_retirement.py b/city_scrapers/spiders/chi_municipal_retirement.py deleted file mode 100644 index 1e1d83a94..000000000 --- a/city_scrapers/spiders/chi_municipal_retirement.py +++ /dev/null @@ -1,90 +0,0 @@ -import re -from datetime import datetime, time - -from city_scrapers_core.constants import BOARD, COMMITTEE -from city_scrapers_core.items import Meeting -from city_scrapers_core.spiders import CityScrapersSpider - - -class ChiMunicipalRetirementSpider(CityScrapersSpider): - name = "chi_municipal_retirement" - agency = "Municipal Employees' Annuity and Benefit Fund of Chicago" - timezone = "America/Chicago" - start_urls = ["https://www.meabf.org/retirement-board/minutes"] - location = { - "name": "Fund Office", - "address": "321 N Clark St, Suite 700, Chicago, IL 60654", - } - - def parse(self, response): - """ - `parse` should always `yield` Meeting items. - - Change the `_parse_title`, `_parse_start`, etc methods to fit your scraping - needs. - """ - self._parse_location(response) - for item in response.css(".data_row:not(.data_head)"): - title = self._parse_title(item) - meeting = Meeting( - title=title, - description="", - classification=self._parse_classification(title), - start=self._parse_start(item), - end=None, - all_day=False, - time_notes="See agenda to confirm time", - location=self.location, - links=self._parse_links(item, response), - source=response.url, - ) - - meeting["status"] = self._get_status(meeting) - meeting["id"] = self._get_id(meeting) - - yield meeting - - def _parse_title(self, item): - """Parse or generate meeting title.""" - title_str = " ".join(item.css("div:nth-child(2)::text").extract()).strip() - if "reg" in title_str.lower(): - return "Retirement Board" - return title_str - - def _parse_classification(self, title): - """Parse or generate classification from allowed options.""" - if "Board" in title: - return BOARD - return COMMITTEE - - def _parse_start(self, item): - """Parse start datetime as a naive datetime object.""" - date_str = " ".join(item.css("div:first-child::text").extract()).strip() - date_obj = datetime.strptime(date_str, "%B %d, %Y").date() - # Default to 9 AM time unless other time shows up in the description - time_obj = time(9) - desc_str = " ".join(item.css("div:nth-child(2)::text").extract()).strip() - time_match = re.search(r"\d{1,2}:\d{1,2} [apmAPM\.]{2,4}", desc_str) - if time_match: - time_str = time_match.group().upper().replace(".", "").strip() - time_obj = datetime.strptime(time_str, "%I:%M %p").time() - return datetime.combine(date_obj, time_obj) - - def _parse_location(self, response): - """Parse or generate location.""" - if "321 N" not in " ".join( - response.css("#content-container p::text").extract() - ): - raise ValueError("Meeting location has changed") - - def _parse_links(self, item, response): - """Parse or generate links.""" - links = [] - for link in item.css("a"): - links.append( - { - "href": response.urljoin(link.attrib["href"]), - "title": " ".join(link.css("*::text").extract()).strip(), - } - ) - return links diff --git a/tests/files/chi_municipal_retirement.html b/tests/files/chi_municipal_retirement.html deleted file mode 100644 index 878a4d3b9..000000000 --- a/tests/files/chi_municipal_retirement.html +++ /dev/null @@ -1,396 +0,0 @@ - - - - - - - - Municipal Employees' Annuity and Benefit Fund of Chicago | Minutes - - - - - - - - - - - - - - - - - - - - - - -
-
- -
-
-
- -
-
-

Board Minutes

-
-
-
- - -
- -
-
- -
-
-
- Meeting Date -
-
- Meeting Description -
-
- Documents -
-
-
-
- April 18, 2019  -
-
- Regular Board Meeting  -
-
- - - Agenda - -   -
-
-
-
- March 26, 2019  -
-
- Investment Meeting  -
-
- - - Agenda - -   -
-
-
-
- March 21, 2019  -
-
- Regualr Board Meeting  -
-
- - - Agenda - -   -
-
-
-
- February 21, 2019  -
-
- Regular Board Meeting  -
-
- - - Agenda - -   -
-
-
-
- January 24, 2019  -
-
- Regular  -
-
- - Minutes - -   -
-
-
- -
- -
-

All meetings begin at 9 AM unless otherwise posted in Agenda.
-Board Meetings Are Held At The Office Of The Fund, located at:
-321 North Clark Street, Suite 700, Chicago, IL 60654  Click here for directions 

-
- - - - -
-
- -
- -
- -
-
-
- -
- - - -
- -
- - - - \ No newline at end of file diff --git a/tests/test_chi_municipal_retirement.py b/tests/test_chi_municipal_retirement.py deleted file mode 100644 index cb9d8ffc4..000000000 --- a/tests/test_chi_municipal_retirement.py +++ /dev/null @@ -1,86 +0,0 @@ -from datetime import datetime -from os.path import dirname, join - -import pytest -from city_scrapers_core.constants import BOARD, COMMITTEE, PASSED, TENTATIVE -from city_scrapers_core.utils import file_response -from freezegun import freeze_time - -from city_scrapers.spiders.chi_municipal_retirement import ChiMunicipalRetirementSpider - -test_response = file_response( - join(dirname(__file__), "files", "chi_municipal_retirement.html"), - url="https://www.meabf.org/retirement-board/minutes", -) -spider = ChiMunicipalRetirementSpider() - -freezer = freeze_time("2019-04-17") -freezer.start() - -parsed_items = [item for item in spider.parse(test_response)] - -freezer.stop() - - -def test_count(): - assert len(parsed_items) == 5 - - -def test_title(): - assert parsed_items[0]["title"] == "Retirement Board" - assert parsed_items[1]["title"] == "Investment Meeting" - - -def test_description(): - assert parsed_items[0]["description"] == "" - - -def test_start(): - assert parsed_items[0]["start"] == datetime(2019, 4, 18, 9) - - -def test_end(): - assert parsed_items[0]["end"] is None - - -def test_time_notes(): - assert parsed_items[0]["time_notes"] == "See agenda to confirm time" - - -def test_id(): - assert ( - parsed_items[0]["id"] - == "chi_municipal_retirement/201904180900/x/retirement_board" - ) - - -def test_status(): - assert parsed_items[0]["status"] == TENTATIVE - assert parsed_items[1]["status"] == PASSED - - -def test_location(): - assert parsed_items[0]["location"] == spider.location - - -def test_source(): - assert parsed_items[0]["source"] == "https://www.meabf.org/retirement-board/minutes" - - -def test_links(): - assert parsed_items[0]["links"] == [ - { - "href": "https://www.meabf.org/assets/pdfs/meetings/agenda/Posted_Agenda_2019-04_REVISED2.pdf", # noqa - "title": "Agenda", - } - ] - - -def test_classification(): - assert parsed_items[0]["classification"] == BOARD - assert parsed_items[1]["classification"] == COMMITTEE - - -@pytest.mark.parametrize("item", parsed_items) -def test_all_day(item): - assert item["all_day"] is False