Skip to content

Commit

Permalink
Merge branch 'main' into tim/2024/al-gql-updates
Browse files Browse the repository at this point in the history
  • Loading branch information
@showerst
showerst committed Dec 18, 2024
2 parents 9f7cd3b + eb547f4 commit 8a47d4e
Show file tree
Hide file tree
Showing 22 changed files with 1,039 additions and 521 deletions.
2 changes: 1 addition & 1 deletion Dockerfile.california
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,4 +63,4 @@ RUN poetry install --extras "california" \

ENV OPENSSL_CONF=/opt/openstates/openstates/openssl.cnf

ENTRYPOINT ["/entrypoint.sh"]
ENTRYPOINT ["/ca-entrypoint.sh"]
4 changes: 2 additions & 2 deletions scrapers/de/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,15 +102,15 @@ class Delaware(State):
"name": "152nd General Assembly (2023-2024)",
"start_date": "2022-01-10",
"end_date": "2024-06-30",
"active": True,
"active": False,
},
{
"_scraped_name": "2024 - 2026 (GA 153)",
"identifier": "153",
"name": "153rd General Assembly (2025-2026)",
"start_date": "2025-01-14",
"end_date": "2026-06-30",
"active": False,
"active": True,
},
]
ignored_scraped_sessions = [
Expand Down
2 changes: 1 addition & 1 deletion scrapers/fl/bills.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ class BillDetail(HtmlPage):
example_source = "https://flsenate.gov/Session/Bill/2021/1"

def get_source_from_input(self):
return self.input.sources[0]["url"]
return URL(self.input.sources[0]["url"], verify=False)

def process_page(self):
if self.root.xpath("//div[@id = 'tabBodyBillHistory']//table"):
Expand Down
78 changes: 45 additions & 33 deletions scrapers/il/__init__.py
View file
Original file line number Diff line number Diff line change
@@ -1,45 +1,42 @@
# encoding=utf-8
from utils import url_xpath
import requests
from openstates.scrape import State
from .bills import IlBillScraper
from .events import IlEventScraper


class Illinois(State):
scrapers = {
"bills": IlBillScraper,
"events": IlEventScraper,
}
scrapers = {"bills": IlBillScraper, "events": IlEventScraper}
legislative_sessions = [
{
"name": "90th Regular Session",
"identifier": "90th",
"classification": "primary",
"_scraped_name": "90 (1997-1998)",
"_scraped_name": "90th General Assembly (1997-1998)",
"start_date": "1997-01-08",
"end_date": "1999-01-12",
},
{
"name": "91st Regular Session",
"identifier": "91st",
"classification": "primary",
"_scraped_name": "91 (1999-2000)",
"_scraped_name": "91st General Assembly (1999-2000)",
"start_date": "1999-01-13",
"end_date": "2001-01-09",
},
{
"name": "92nd Regular Session",
"identifier": "92nd",
"classification": "primary",
"_scraped_name": "92 (2001-2002)",
"_scraped_name": "92nd General Assembly (2001-2002)",
"start_date": "2001-01-10",
"end_date": "2003-01-07",
},
{
"name": "93rd Regular Session",
"identifier": "93rd",
"classification": "primary",
"_scraped_name": "93 (2003-2004)",
"_scraped_name": "93rd General Assembly (2003-2004)",
"start_date": "2003-01-08",
"end_date": "2005-01-11",
},
Expand All @@ -54,15 +51,15 @@ class Illinois(State):
"name": "94th Regular Session",
"identifier": "94th",
"classification": "primary",
"_scraped_name": "94 (2005-2006)",
"_scraped_name": "94th General Assembly (2005-2006)",
"start_date": "2005-01-12",
"end_date": "2007-01-09",
},
{
"name": "95th Regular Session",
"identifier": "95th",
"classification": "primary",
"_scraped_name": "95 (2007-2008)",
"_scraped_name": "95th General Assembly (2007-2008)",
"start_date": "2007-01-10",
"end_date": "2009-01-13",
},
Expand All @@ -77,7 +74,7 @@ class Illinois(State):
"name": "96th Regular Session",
"identifier": "96th",
"classification": "primary",
"_scraped_name": "96 (2009-2010)",
"_scraped_name": "96th General Assembly (2009-2010)",
"start_date": "2009-01-14",
"end_date": "2011-01-11",
},
Expand All @@ -92,31 +89,31 @@ class Illinois(State):
"name": "97th Regular Session",
"identifier": "97th",
"classification": "primary",
"_scraped_name": "97 (2011-2012)",
"_scraped_name": "97th General Assembly (2011-2012)",
"start_date": "2011-01-12",
"end_date": "2013-01-08",
},
{
"name": "98th Regular Session",
"identifier": "98th",
"classification": "primary",
"_scraped_name": "98 (2013-2014)",
"_scraped_name": "98th General Assembly (2013-2014)",
"start_date": "2013-01-09",
"end_date": "2015-01-13",
},
{
"name": "99th Regular Session",
"identifier": "99th",
"classification": "primary",
"_scraped_name": "99 (2015-2016)",
"_scraped_name": "99th General Assembly (2015-2016)",
"start_date": "2015-01-14",
"end_date": "2017-01-10",
},
{
"name": "100th Special Session",
"identifier": "100th-special",
"classification": "special",
"_scraped_name": "100 (2017-2018)",
"_scraped_name": "100th General Assembly (2017-2018)",
"start_date": "2017-06-21",
"end_date": "2017-06-21",
},
Expand All @@ -133,12 +130,12 @@ class Illinois(State):
"start_date": "2019-01-09",
"end_date": "2019-12-14",
"classification": "primary",
"_scraped_name": "101 (2019-2020)",
"_scraped_name": "101st General Assembly (2019-2020)",
},
# Leave this on until 2023-01-31,
# IL has a history post-session governor actions
{
"_scraped_name": "102 (2021-2022)",
"_scraped_name": "102nd General Assembly (2021-2022)",
"name": "102nd Regular Session",
"identifier": "102nd",
"start_date": "2021-01-13",
Expand All @@ -148,7 +145,16 @@ class Illinois(State):
},
# check senate prez in session_details in bills.py
# https://www.ilga.gov/house/schedules/2024_House_Spring_Session.pdf
{
# {
# "name": "103rd Regular Session",
# "identifier": "103rd",
# "start_date": "2023-01-11",
# "end_date": "2024-05-24",
# "classification": "primary",
# "active": False,
# },
{
"_scraped_name": "103rd General Assembly (2023-2024)",
"name": "103rd Regular Session",
"identifier": "103rd",
"start_date": "2023-01-11",
Expand All @@ -159,20 +165,26 @@ class Illinois(State):
]

ignored_scraped_sessions = [
"77 (1971-1972)",
"78 (1973-1974)",
"79 (1975-1976)",
"80 (1977-1978)",
"81 (1979-1980)",
"82 (1981-1982)",
"83 (1983-1984)",
"84 (1985-1986)",
"85 (1987-1988)",
"86 (1989-1990)",
"87 (1991-1992)",
"88 (1993-1994)",
"89 (1995-1996)",
"89th General Assembly (1995-1996)",
"88th General Assembly (1993-1994)",
"87th General Assembly (1991-1992)",
"86th General Assembly (1989-1990)",
"85th General Assembly (1987-1988)",
"84th General Assembly (1985-1986)",
"83rd General Assembly (1983-1984)",
"82nd General Assembly (1981-1982)",
"81st General Assembly (1979-1980)",
"80th General Assembly (1977-1978)",
"79th General Assembly (1975-1976)",
"78th General Assembly (1973-1974)",
"77th General Assembly (1971-1972)",
]

def get_session_list(self):
return url_xpath("https://ilga.gov/PreviousGA.asp", "//option/text()")
response = requests.get(
"https://beta.ilga.gov/API/Legislation/GetGeneralAssemblies"
)
response.raise_for_status()
session_list = [ga["gaLabel"] for ga in response.json()]

return session_list
Loading

0 comments on commit 8a47d4e

Please sign in to comment.