diff --git a/city_scrapers/spiders/il_lottery.py b/city_scrapers/spiders/il_lottery.py deleted file mode 100644 index fa0886e91..000000000 --- a/city_scrapers/spiders/il_lottery.py +++ /dev/null @@ -1,205 +0,0 @@ -import re -from collections import defaultdict -from datetime import datetime, time - -from city_scrapers_core.constants import BOARD -from city_scrapers_core.items import Meeting -from city_scrapers_core.spiders import CityScrapersSpider - - -class IlLotterySpider(CityScrapersSpider): - name = "il_lottery" - agency = "Illinois Lottery Control Board" - timezone = "America/Chicago" - start_urls = [ - "https://www.illinoislottery.com/illinois-lottery/lottery-control-board/" - ] - - def parse(self, response): - """ - `parse` should always `yield` Meeting items. - - Change the `_parse_title`, `_parse_start`, etc methods to fit your scraping - needs. - """ - - # if "122 South Michigan Avenue, 19th Floor" not in response.text: - # raise ValueError("Meeting address has changed") - - upcoming_meetings = self._parse_upcoming_meetings(response) - past_meetings = self._parse_past_meetings(response) - links = self._parse_links(response) - - # create a master dictionary with one key per meeting date - reconciled_meetings = upcoming_meetings - only_past_dates = set(past_meetings.keys()).difference(upcoming_meetings.keys()) - only_past_meetings = { - key: value for key, value in past_meetings.items() if key in only_past_dates - } - reconciled_meetings.update(only_past_meetings) - - for key in sorted(reconciled_meetings.keys(), reverse=True): - item = reconciled_meetings[key] - meeting = Meeting( - title=self._parse_title(item), - description="", - classification=BOARD, - start=self._parse_start(item), - end=None, - all_day=False, - time_notes="", - location={"name": "Virtual", "address": ""}, - # location={ - # "name": "Chicago Lottery Office", - # "address": "122 South Michigan Avenue, 19th Floor, Chicago, IL 60603", # noqa - # }, - source=response.url, - ) - meeting["id"] = self._get_id(meeting) - meeting["status"] = self._get_status(meeting, text=item) - meeting["links"] = links.get(meeting["start"].date(), []) - - yield meeting - - def _parse_title(self, item): - """Parse or generate meeting title.""" - suffix = "" - if "QTR" in item: - suffix = "Quarterly " - elif "Special" in item: - suffix = "Special " - return "Lottery Control Board {}Meeting".format(suffix) - - @staticmethod - def parse_time(source): - """Returns times given string with format h:mm am|pm""" - time_regex = re.compile(r"([1-9]:[0-5][0-9]\s?[a|p]m)") - - try: - time_match = time_regex.search(source).group().replace(" ", "") - return datetime.strptime(time_match, "%I:%M%p").time() - except AttributeError: - default_hour = 13 - default_minute = 30 - return time(default_hour, default_minute) - - @staticmethod - def parse_day(source): - """Returns date""" - # search for dates with '/' (ex: 08/16/19) - if "/" in source: - date_match = re.search(r"(\d{2}\/\d{2}\/\d{2})", source).group() - dt = datetime.strptime(date_match, "%m/%d/%y").date() - # search for date in format "[month] [dd], [yyyy]" (ex: 'May 15, 2019') - else: - date_match = re.search(r"[a-zA-Z]{3,10} \d{1,2}?,? \d{4}", source).group() - dt = datetime.strptime(date_match, "%B %d, %Y").date() - - return dt - - def _parse_start(self, item): - """Parse start date and time.""" - return datetime.combine(self.parse_day(item), self.parse_time(item)) - - def _parse_links(self, response, link_text_substrings=["Agenda", "Minutes"]): - """ - Extracts link to agenda for a specific meeting date - Args: - date: A datetime object with the date of the desired agenda - link_text_substrings: A list with strings that must exist in link text - in order to be added to link dictionary - Return: - List of dictionaries with the keys title (title/description of the link) - and href (link URL) for the agenda for the meeting on the requested date - """ - link_date_map = defaultdict(list) - for link in response.xpath("//p//a"): - link_text_selector = link.xpath("text()") - if link_text_selector: - link_text = link_text_selector.get() - if any( - link_text_substring in link_text - for link_text_substring in link_text_substrings - ): - link_date = self.parse_day(link_text) - link_href = link.xpath("@href").get() - link_date_map[link_date].append( - { - "href": response.urljoin(link_href), - "title": link_text.replace("\xa0", " "), - } - ) - - return link_date_map - - def _parse_upcoming_meetings(self, response): - """ - Returns a list of lines with dates following the string 'Upcoming Meeting Dates' - """ - - meeting_xpath = '//p[contains(., "Upcoming meeting dates")]' - - # future meetings are separated by
tags - meeting_lines = response.xpath(meeting_xpath).css("p *::text").extract() - - special_meeting_xpath = '//p[contains(., "Special meeting of")]' - special_meeting_lines = ( - response.xpath(special_meeting_xpath).css("p *::text").extract() - ) - # only keep lines that include a weekday - weekdays = [ - "Monday", - "Tuesday", - "Wednesday", - "Thursday", - "Friday", - "Saturday", - "Sunday", - ] - meeting_lines = [ - x for x in meeting_lines if any(weekday in x for weekday in weekdays) - ] - special_meeting_lines = [ - "Special " + line - for line in special_meeting_lines - if any(weekday in line for weekday in weekdays) - ] - - meetings = {} - for meeting_text in meeting_lines + special_meeting_lines: - meeting_date = self.parse_day(meeting_text) - meetings = self._update_meeting_value(meetings, meeting_date, meeting_text) - - return meetings - - def _parse_past_meetings(self, response): - """Returns a list of start date and documents from meeting minutes page""" - meetings = {} - for item in response.css("a"): - meeting_text = item.xpath("text()").get() - if meeting_text is not None: - if "Agenda" in meeting_text or "Minutes" in meeting_text: - meeting_date = self.parse_day(meeting_text) - meetings = self._update_meeting_value( - meetings, meeting_date, meeting_text - ) - return meetings - - def _update_meeting_value(self, meetings, meeting_date, meeting_text): - """ - Updates the meetings dictionary - If date does not yet exist in dictionary, adds to dictionary - If date already exists, updates value with additional text found for the meeting - - Args: - meetings: Dict where key = date and value = text associated with the meeting - meeting_date: A datetime object representing a meeting date - meeting_text: A string with information about a meeting - Return: - An updated version of the meetings dictionary - """ - if meetings.get(meeting_date) is None: - meetings[meeting_date] = meeting_text - else: - meetings[meeting_date] = meetings[meeting_date] + " " + meeting_text - return meetings diff --git a/tests/files/il_lottery.html b/tests/files/il_lottery.html deleted file mode 100644 index d4799c366..000000000 --- a/tests/files/il_lottery.html +++ /dev/null @@ -1,1113 +0,0 @@ - - - - - Lottery Control Board | Members and Meetings | Illinois Lottery - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- - - - - - -
- -
- - - - - Skip to main content - -
-
- - - -
-
- - - - - -
- -
-
-
Games
- Check them out -
- -
- POWERBALL -
- - -
- MEGA MILLIONS -
- - -
- LOTTO -
- - -
- LUCKY DAY LOTTO -
- - -
- PICK 3 -
- - -
- PICK 4 -
- - -
-

Instant Tickets

-
- Discover All -
-
- - - - - Results - - - - - Winning - - - - - Giving Back - - - - My Account - -
- - - Illinois Lottery - - - About the Games - - - Useful Information - - - Help Center - - -
- -
- - - - - -
-
- - - -
-
- - -
- - -
- -
-
-
- -
-
- -
- - - - -
- - - - -
-
- - -
-
-
- -
-
- -
-
- -
- Skip the forthcoming navigation - - - - - - - - -
- - -
- - -
-
-
- -
- - -
-

Lottery Control Board

-
-
- - - - - -
-
-
-

The Lottery Control Board advises the Lottery Director and Department of Lottery.

-

Board Members

- -
- - - -
-
-
-
- - - - - - - - - - - - - - - - -
NameTerm ExpiresMember Since
Elba Aranda-Suh07/01/201707/06/2015
James Floyd07/01/201903/20/2017
Nimish Jani07/01/202009/21/2018
Lori Yokoyama07/01/202110/31/2018
- -
-
- - - -
-
-
-

Next Meeting Notice
-

-

Lottery Control Board Meetings are held at:
- 122 South Michigan Avenue, 19th Floor
- Chicago, Illinois 60603
-
- Call-in information is available upon request to the Lottery Control Board Secretary.
-

-

PUBLIC NOTICE MEETING DATE CHANGE: Notice is hereby given that the regular quarter 2 meeting of the Lottery Control Board, scheduled at 1:30 p.m. on Wednesday  May 8, 2019 has been rescheduled to 1:30 p.m. on Wednesday, May 15, 2019.  The meeting will be held at the Chicago Lottery office located at 122 S. Michigan Avenue, 19th Floor, Chicago, Illinois.

-


- Upcoming meeting dates:
- 2nd QTR Meeting - Wednesday, May 15, 2019 1:30pm
- 3rd QTR Meeting - Wednesday, August 21, 2019 1:30pm
- 4th QTR Meeting - Wednesday, November 6, 2019 1:30pm
- Special Meeting of the Lottery Control Board - Friday, 08/16/19 at 3:00 pm.

-

Illinois Lottery Control Board Meeting Agenda - May 15, 2019
- Special Meeting of the Lottery Control Board Agenda - August 16, 2019

-

 

-

Previous Meeting Notices

-

Illinois Lottery Control Board Meeting Agenda - February 6, 2019

-

Illinois Lottery Control Board Meeting Agenda - December 5, 2018

-

Illinois Lottery Control Board Meeting Agenda - June 6, 2018

-

Illinois Lottery Control Board Meeting Agenda-February 26, 2018

-

Illinois Lottery Control Board Meeting Agenda-December 8, 2017

-

Illinois Lottery Control Board Meeting Agenda-September 8, 2017

-

Illinois Lottery Control Board Meeting Agenda-June 8, 2017

-

Illinois Lottery Control Board Meeting Agenda-March 2, 2017

-

Illinois Lottery Control Board Meeting Agenda-December 1, 2016
-

-

Illinois Lottery Control Board Meeting Agenda-August 30, 2016
-

-

Illinois Lottery Control Board Meeting Agenda-February 19, 2015

-

Illinois Lottery Control Board Meeting Agenda-May 21, 2014 at 9:30 AM

-

Illinois Lottery Control Board Meeting Agenda-March 12, 2014 at 9:30 AM

-

 

-

Minutes from Previous Meetings

-

Illinois Lottery Control Board Meeting Minutes-February 6, 2019

-

Illinois Lottery Control Board Meeting Minutes-December 05, 2018

-

Illinois Lottery Control Board Meeting Minutes-June 06, 2018

-

Illinois Lottery Control Board Meeting Minutes-February 26, 2018

-

Illinois Lottery Control Board Meeting Minutes-December 08, 2017
-

-

Illinois Lottery Control Board Meeting Minutes-September 08, 2017
-

-

Illinois Lottery Control Board Meeting Minutes-June 08, 2017
-

-

Illinois Lottery Control Board Meeting Minutes-March 02, 2017
-

-

Illinois Lottery Control Board Meeting Minutes-December 01, 2016
-

-

Illinois Lottery Control Board Meeting Minutes-August 30, 2016
-

-

Illinois Lottery Control Board Meeting Minutes-February 19, 2015
-

-

Illinois Lottery Control Board Meeting Minutes-May 21, 2014
-

-

Illinois Lottery Control Board Meeting Minutes-March 12, 2014
-

-

Illinois Lottery Control Board Meeting Minutes-November 13, 2013
-

-

Illinois Lottery Control Board Meeting Minutes-October 17, 2013
-

-

Illinois Lottery Control Board Meeting Minutes-September 27, 2013
-

-

Illinois Lottery Control Board Meeting Minutes-August 07, 2013
-

-

Illinois Lottery Control Board Meeting Minutes-April 17, 2013

-

Illinois Lottery Control Board Meeting Minutes-January 23, 2013
-

-

Illinois Lottery Control Board Meeting Minutes-October 10, 2012

-

 

-

Lottery Control Board Annual Reports

-

Lottery Control Board FY18 Annual Report

-

Lottery Control Board FY19 Annual Report

-

 

-

Lottery Control Board Bylaw Amendment

-

At the March 2014 Lottery Control Board meeting, the Board approved an amendment to the bylaws.
-

-

Section 9 Public Record and Public Participation was amended as follows:
-

-

(e) This Board will permit any member of the public to speak on any matter listed on the meeting’s publicly posted agenda. In accordance with 5 ILCS 120/2.06(g), this Board requires that persons wishing to comment on agenda items during a public meeting of the Board, request permission to speak in writing, at least 24 hours before the meeting. In the event that the meeting immediately follows a weekend or holiday, any written request to address the Board must be received no later than 4:00pm on the last business day preceding the weekend or holiday. Such requests should be submitted in writing to the Secretary of the Board, and should provide the following information:
-

-

 

-

1. Speaker’s Name
- 2. Name of Business, Association or Organization, if applicable
- 3. Residential or Business Address
- 4. Agenda topic(s) on which the person wishes to address the Board.
-

-

I. Designated comment period Public comments are the first items addressed on each Board meeting agenda. The chair may require persons wishing to speak to sign in before the start of the meeting and to provide their names and business or residential address. Prior to speaking, each person must be recognized by the chair and must state and spell his or her name for the public record.

-

II. Time limits Public comment is limited to no more than three minutes per person and to no more than fifteen minutes in total per meeting, unless there is consent of a majority vote of the members present to extend the time period. The chair shall monitor each speaker’s use of time and shall notify the speaker when the time allotted has expired.

-

III. Limits on group comments Whenever any group of persons wishes to address the Board on the same topic, the chair may ask that a spokesperson be chosen from the group. If additional matters are to be presented by other persons in the group, the chair may limit the number of such persons and may limit the presentation to information not already presented by the group spokesperson.

-

 

-

Note: The Secretary of the board can be reached at LOT.LCBSecretary@Illinois.gov or via US Mail at 122 S. Michigan Avenue, 19th Floor, Chicago, IL 60603.

- -
- - - -
- - -
- - -
-
-
- - - -
- - -
-
- - - - - - -
- -
- - -
- -
- - - - - - - - - - - - - - - - - - - -
- - - - - - - - - diff --git a/tests/test_il_lottery.py b/tests/test_il_lottery.py deleted file mode 100644 index 5ffc10a85..000000000 --- a/tests/test_il_lottery.py +++ /dev/null @@ -1,84 +0,0 @@ -from datetime import datetime -from os.path import dirname, join - -import pytest -from city_scrapers_core.constants import BOARD, PASSED, TENTATIVE -from city_scrapers_core.utils import file_response -from freezegun import freeze_time - -from city_scrapers.spiders.il_lottery import IlLotterySpider - -test_response = file_response( - join(dirname(__file__), "files", "il_lottery.html"), - url="https://www.illinoislottery.com/illinois-lottery/lottery-control-board", -) -spider = IlLotterySpider() - -freezer = freeze_time("2019-08-17") -freezer.start() - -parsed_items = [item for item in spider.parse(test_response)] - -freezer.stop() - - -def test_len(): - assert len(parsed_items) == 24 - - -def test_title(): - assert parsed_items[0]["title"] == "Lottery Control Board Quarterly Meeting" - assert parsed_items[2]["title"] == "Lottery Control Board Special Meeting" - - -@pytest.mark.parametrize("item", parsed_items) -def test_description(item): - assert item["description"] == "" - - -def test_start(): - assert parsed_items[0]["start"] == datetime(2019, 11, 6, 13, 30) - - -@pytest.mark.parametrize("item", parsed_items) -def test_time_notes(item): - assert item["time_notes"] == "" - - -def test_status(): - assert parsed_items[0]["status"] == TENTATIVE - assert parsed_items[2]["status"] == PASSED - - -# @pytest.mark.parametrize("item", parsed_items) -# def test_location(item): -# assert item["location"] == { -# "name": "Chicago Lottery Office", -# "address": "122 South Michigan Avenue, 19th Floor, Chicago, IL 60603", -# } - - -@pytest.mark.parametrize("item", parsed_items) -def test_source(item): - source = "https://www.illinoislottery.com/illinois-lottery/lottery-control-board" - assert item["source"] == source - - -def test_links(): - assert len(parsed_items[0]["links"]) == 0 - assert parsed_items[3]["links"] == [ - { - "href": "https://www.illinoislottery.com/content/dam/il/pdfs/lottery-control-board/May%202019%20Lottery%20Control%20Board%20Meeting%20Agenda.pdf", # noqa - "title": "Illinois Lottery Control Board Meeting Agenda - May 15, 2019", - } - ] - - -@pytest.mark.parametrize("item", parsed_items) -def test_classification(item): - assert item["classification"] == BOARD - - -@pytest.mark.parametrize("item", parsed_items) -def test_all_day(item): - assert item["all_day"] is False