Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

IL: Bill scraper rewrite to new beta site #5121

Merged
merged 4 commits into from
Dec 13, 2024
Merged

IL: Bill scraper rewrite to new beta site #5121

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
4e5a263
IL: Bill scraper rewrite to new beta site
braykuka Dec 9, 2024
5279c5c
fix: lint issue
braykuka Dec 9, 2024
e0190f8
Fix small issue
braykuka Dec 10, 2024
6437443
fix: no abstract issue
braykuka Dec 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78 changes: 45 additions & 33 deletions scrapers/il/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,45 +1,42 @@
# encoding=utf-8
from utils import url_xpath
import requests
from openstates.scrape import State
from .bills import IlBillScraper
from .events import IlEventScraper


class Illinois(State):
scrapers = {
"bills": IlBillScraper,
"events": IlEventScraper,
}
scrapers = {"bills": IlBillScraper, "events": IlEventScraper}
legislative_sessions = [
{
"name": "90th Regular Session",
"identifier": "90th",
"classification": "primary",
"_scraped_name": "90 (1997-1998)",
"_scraped_name": "90th General Assembly (1997-1998)",
"start_date": "1997-01-08",
"end_date": "1999-01-12",
},
{
"name": "91st Regular Session",
"identifier": "91st",
"classification": "primary",
"_scraped_name": "91 (1999-2000)",
"_scraped_name": "91st General Assembly (1999-2000)",
"start_date": "1999-01-13",
"end_date": "2001-01-09",
},
{
"name": "92nd Regular Session",
"identifier": "92nd",
"classification": "primary",
"_scraped_name": "92 (2001-2002)",
"_scraped_name": "92nd General Assembly (2001-2002)",
"start_date": "2001-01-10",
"end_date": "2003-01-07",
},
{
"name": "93rd Regular Session",
"identifier": "93rd",
"classification": "primary",
"_scraped_name": "93 (2003-2004)",
"_scraped_name": "93rd General Assembly (2003-2004)",
"start_date": "2003-01-08",
"end_date": "2005-01-11",
},
Expand All @@ -54,15 +51,15 @@ class Illinois(State):
"name": "94th Regular Session",
"identifier": "94th",
"classification": "primary",
"_scraped_name": "94 (2005-2006)",
"_scraped_name": "94th General Assembly (2005-2006)",
"start_date": "2005-01-12",
"end_date": "2007-01-09",
},
{
"name": "95th Regular Session",
"identifier": "95th",
"classification": "primary",
"_scraped_name": "95 (2007-2008)",
"_scraped_name": "95th General Assembly (2007-2008)",
"start_date": "2007-01-10",
"end_date": "2009-01-13",
},
Expand All @@ -77,7 +74,7 @@ class Illinois(State):
"name": "96th Regular Session",
"identifier": "96th",
"classification": "primary",
"_scraped_name": "96 (2009-2010)",
"_scraped_name": "96th General Assembly (2009-2010)",
"start_date": "2009-01-14",
"end_date": "2011-01-11",
},
Expand All @@ -92,31 +89,31 @@ class Illinois(State):
"name": "97th Regular Session",
"identifier": "97th",
"classification": "primary",
"_scraped_name": "97 (2011-2012)",
"_scraped_name": "97th General Assembly (2011-2012)",
"start_date": "2011-01-12",
"end_date": "2013-01-08",
},
{
"name": "98th Regular Session",
"identifier": "98th",
"classification": "primary",
"_scraped_name": "98 (2013-2014)",
"_scraped_name": "98th General Assembly (2013-2014)",
"start_date": "2013-01-09",
"end_date": "2015-01-13",
},
{
"name": "99th Regular Session",
"identifier": "99th",
"classification": "primary",
"_scraped_name": "99 (2015-2016)",
"_scraped_name": "99th General Assembly (2015-2016)",
"start_date": "2015-01-14",
"end_date": "2017-01-10",
},
{
"name": "100th Special Session",
"identifier": "100th-special",
"classification": "special",
"_scraped_name": "100 (2017-2018)",
"_scraped_name": "100th General Assembly (2017-2018)",
"start_date": "2017-06-21",
"end_date": "2017-06-21",
},
Expand All @@ -133,12 +130,12 @@ class Illinois(State):
"start_date": "2019-01-09",
"end_date": "2019-12-14",
"classification": "primary",
"_scraped_name": "101 (2019-2020)",
"_scraped_name": "101st General Assembly (2019-2020)",
},
# Leave this on until 2023-01-31,
# IL has a history post-session governor actions
{
"_scraped_name": "102 (2021-2022)",
"_scraped_name": "102nd General Assembly (2021-2022)",
"name": "102nd Regular Session",
"identifier": "102nd",
"start_date": "2021-01-13",
Expand All @@ -148,7 +145,16 @@ class Illinois(State):
},
# check senate prez in session_details in bills.py
# https://www.ilga.gov/house/schedules/2024_House_Spring_Session.pdf
{
# {
# "name": "103rd Regular Session",
# "identifier": "103rd",
# "start_date": "2023-01-11",
# "end_date": "2024-05-24",
# "classification": "primary",
# "active": False,
# },
{
"_scraped_name": "103rd General Assembly (2023-2024)",
"name": "103rd Regular Session",
"identifier": "103rd",
"start_date": "2023-01-11",
Expand All @@ -159,20 +165,26 @@ class Illinois(State):
]

ignored_scraped_sessions = [
"77 (1971-1972)",
"78 (1973-1974)",
"79 (1975-1976)",
"80 (1977-1978)",
"81 (1979-1980)",
"82 (1981-1982)",
"83 (1983-1984)",
"84 (1985-1986)",
"85 (1987-1988)",
"86 (1989-1990)",
"87 (1991-1992)",
"88 (1993-1994)",
"89 (1995-1996)",
"89th General Assembly (1995-1996)",
"88th General Assembly (1993-1994)",
"87th General Assembly (1991-1992)",
"86th General Assembly (1989-1990)",
"85th General Assembly (1987-1988)",
"84th General Assembly (1985-1986)",
"83rd General Assembly (1983-1984)",
"82nd General Assembly (1981-1982)",
"81st General Assembly (1979-1980)",
"80th General Assembly (1977-1978)",
"79th General Assembly (1975-1976)",
"78th General Assembly (1973-1974)",
"77th General Assembly (1971-1972)",
]

def get_session_list(self):
return url_xpath("https://ilga.gov/PreviousGA.asp", "//option/text()")
response = requests.get(
"https://beta.ilga.gov/API/Legislation/GetGeneralAssemblies"
)
response.raise_for_status()
session_list = [ga["gaLabel"] for ga in response.json()]

return session_list
Loading
Loading