diff --git a/city_scrapers/spiders/il_opioid_response.py b/city_scrapers/spiders/il_opioid_response.py deleted file mode 100644 index e2f6fc509..000000000 --- a/city_scrapers/spiders/il_opioid_response.py +++ /dev/null @@ -1,133 +0,0 @@ -import re -from datetime import datetime, time - -from city_scrapers_core.constants import ADVISORY_COMMITTEE -from city_scrapers_core.items import Meeting -from city_scrapers_core.spiders import CityScrapersSpider - - -# TODO: Fix -class IlOpioidResponseSpider(CityScrapersSpider): - name = "il_opioid_response" - agency = "Illinois Opioid Crisis Response Advisory Council" - timezone = "America/Chicago" - start_urls = ["https://www.dhs.state.il.us/page.aspx?item=104682"] - - def parse(self, response): - """ - `parse` should always `yield` Meeting items. - - Change the `_parse_id`, `_parse_name`, etc methods to fit your scraping - needs. - """ - - for item in response.xpath("//p//a"): - if "Agenda" not in item.xpath("text()").get(): - break - - meeting = Meeting( - title=self._parse_title(item), - description=self._parse_description(item), - classification=self._parse_classification(item), - start=self._parse_start(item), - end=self._parse_end(item), - all_day=self._parse_all_day(item), - time_notes=self._parse_time_notes(item), - location=self._parse_location(item), - links=self._parse_links(item, response), - source=self._parse_source(response), - ) - - meeting["status"] = self._get_status(meeting) - meeting["id"] = self._get_id(meeting) - - yield meeting - - def _parse_title(self, item): - """Parse or generate meeting title.""" - return "Advisory Council" - - def _parse_description(self, item): - """Parse or generate meeting description.""" - return "" - - def _parse_classification(self, item): - """Parse or generate classification from allowed options.""" - return ADVISORY_COMMITTEE - - def _parse_start(self, item): - """Parse start datetime as a naive datetime object.""" - date_match = self._get_date(item) - if date_match: - start_date = datetime.strptime(date_match.group(0), "%m.%d.%y") - return datetime.combine(start_date, time(13)) - else: - return None - - def _parse_end(self, item): - """Parse end datetime as a naive datetime object. Added by pipeline if None""" - date_match = self._get_date(item) - if date_match: - end_date = datetime.strptime(date_match.group(0), "%m.%d.%y") - return datetime.combine(end_date, time(15)) - else: - return None - - def _parse_time_notes(self, item): - """Parse any additional notes on the timing of the meeting""" - return "" - - def _parse_all_day(self, item): - """Parse or generate all-day status. Defaults to False.""" - return False - - def _parse_location(self, item): - """Parse or generate location.""" - return { - "address": "401 S. Clinton Street, 7th Floor Executive Conference Room," - " Chicago, IL 60607", - "name": "Illinois Department of Human Services Clinton Building", - } - - def _parse_links(self, item, response): - """Parse or generate links for meeting agenda and minutes.""" - links = [] - agenda = { - "href": response.urljoin(item.xpath("@href").get()), - "title": item.xpath("text()").get(), - } - - links.append(agenda) - - date_match = self._get_date(item) - - if date_match: - datestr = date_match.group(0) - date_indices = [ - i - for i, x in enumerate(response.xpath("//p//a/text()").getall()) - if datestr in x - ] - - if len(date_indices) > 1: - minutes_idx = date_indices[1] - minutes = response.xpath("//p//a")[minutes_idx] - - if datestr in minutes.get(): - minutes = { - "href": response.urljoin(minutes.xpath("@href").get()), - "title": minutes.xpath("text()").get(), - } - - links.append(minutes) - - return links - - def _parse_source(self, response): - """Parse or generate source.""" - return response.url - - def _get_date(self, item): - text_str = item.xpath("text()").get() - date_match = re.search(r"\d*\.\d*\.\d*", text_str) - return date_match diff --git a/tests/files/il_opioid_response.html b/tests/files/il_opioid_response.html deleted file mode 100644 index 1cf463008..000000000 --- a/tests/files/il_opioid_response.html +++ /dev/null @@ -1,170 +0,0 @@ - - - - - - - IDHS: Advisory Council Agenda & Minutes Archive - - - - - - - - - - - - - - - - - -
-
- -
- - -
-

Advisory Council Agenda & Minutes Archive

-
-
Helping Families. Supporting Communities. Empowering Individuals.
-
-
- - - -
-
-
-
-
-
- - \ No newline at end of file diff --git a/tests/test_il_opioid_response.py b/tests/test_il_opioid_response.py deleted file mode 100644 index 271c328ab..000000000 --- a/tests/test_il_opioid_response.py +++ /dev/null @@ -1,81 +0,0 @@ -from datetime import datetime -from os.path import dirname, join - -import pytest -from city_scrapers_core.constants import ADVISORY_COMMITTEE -from city_scrapers_core.utils import file_response -from freezegun import freeze_time - -from city_scrapers.spiders.il_opioid_response import IlOpioidResponseSpider - -test_response = file_response( - join(dirname(__file__), "files", "il_opioid_response.html"), - url="https://www.dhs.state.il.us/page.aspx?item=97186", -) -spider = IlOpioidResponseSpider() - -freezer = freeze_time("2019-04-16") -freezer.start() - -parsed_items = [item for item in spider.parse(test_response)] - -freezer.stop() - - -def test_title(): - assert parsed_items[0]["title"] == "Advisory Council" - - -def test_description(): - assert parsed_items[0]["description"] == "" - - -def test_start(): - assert parsed_items[0]["start"] == datetime(2019, 4, 15, 13, 0) - - -def test_end(): - assert parsed_items[0]["end"] == datetime(2019, 4, 15, 15, 0) - - -def test_time_notes(): - assert parsed_items[0]["time_notes"] == "" - - -def test_id(): - assert parsed_items[0]["id"] == "il_opioid_response/201904151300/x/advisory_council" - - -def test_status(): - assert parsed_items[0]["status"] == "passed" - - -def test_location(): - assert parsed_items[0]["location"] == { - "address": "401 S. Clinton Street, 7th Floor Executive Conference Room, Chicago, IL 60607", # noqa - "name": "Illinois Department of Human Services Clinton Building", - } - - -def test_source(): - assert ( - parsed_items[0]["source"] == "https://www.dhs.state.il.us/page.aspx?item=97186" - ) - - -def test_links(): - assert parsed_items[0]["links"] == [ - { - "href": "https://www.dhs.state.il.us/OneNetLibrary/27896/documents/Agenda_04.15.19.pdf", # noqa - "title": "Agenda 04.15.19 (pdf)", - } - ] - - -def test_classification(): - assert parsed_items[0]["classification"] == ADVISORY_COMMITTEE - - -@pytest.mark.parametrize("item", parsed_items) -def test_all_day(item): - assert item["all_day"] is False