Merge pull request #4871 from braykuka/1169-al-add-bills-to-events

AL: Add Bills to Events
openstates · Mar 7, 2024 · cf7824f · cf7824f
2 parents 8c37d32 + 1608c95
commit cf7824f
Showing 1 changed file with 25 additions and 5 deletions.
diff --git a/scrapers/al/events.py b/scrapers/al/events.py
@@ -2,7 +2,6 @@
 import dateutil.parser
 import json
 import pytz
-import re
 
 from utils import LXMLMixin
 from utils.events import match_coordinates
@@ -16,7 +15,7 @@ class ALEventScraper(Scraper, LXMLMixin):
     _DATETIME_FORMAT = "%m/%d/%Y %I:%M %p"
 
     def scrape(self, start=None):
-        gql_url = "https://gql.api.alison.legislature.state.al.us/graphql"
+        gql_url = "https://alison.legislature.state.al.us/graphql/"
 
         headers = {
             "Accept": "*/*",
@@ -32,7 +31,7 @@ def scrape(self, start=None):
             start = datetime.datetime.today().replace(day=1).strftime("%Y-%m-%d")
 
         query = (
-            '{hearingsMeetings(eventType:"meeting", body:"", keyword:"", toDate:"3000-02-06", '
+            '{hearingsMeetings(eventType:"meeting", body:"", keyword:"", toDate:"3000-12-31", '
             f'fromDate:"{start}", sortTime:"", direction:"ASC", orderBy:"SortTime", )'
             "{ EventDt,EventTm,Location,EventTitle,EventDesc,Body,DeadlineDt,PublicHearing,"
             "Committee,AgendaUrl,SortTime,OidMeeting,LiveStream }}"
@@ -50,12 +49,34 @@ def scrape(self, start=None):
         if len(page["data"]["hearingsMeetings"]) == 0:
             raise EmptyScrape
 
+        query = (
+            '{hearingsMeetingsDetails(eventType:"meeting", body:"", keyword:"", toDate:"3000-12-31", '
+            f'fromDate:"{start}", sortTime:"", direction:"ASC", orderBy:"SortTime", )'
+            "{EventDt,EventTm,Location,EventTitle,EventDesc,Body,DeadlineDt,PublicHearing,"
+            "LiveStream,Committee,AgendaUrl,SortTime,OidMeeting, Sponsor, InstrumentNbr, ShortTitle, "
+            "OidInstrument, SessionType, SessionYear}}"
+        )
+        json_data = {
+            "query": query,
+            "operationName": "",
+            "variables": [],
+        }
+        details = self.post(gql_url, headers=headers, json=json_data)
+        details = json.loads(details.content)
+
+        bills = {}
+        for row in details["data"]["hearingsMeetingsDetails"]:
+            if row["OidMeeting"] not in bills:
+                bills[row["OidMeeting"]] = []
+            bills[row["OidMeeting"]].append(row["InstrumentNbr"])
+
         event_keys = set()
 
         for row in page["data"]["hearingsMeetings"]:
             event_date = self._TZ.localize(dateutil.parser.parse(row["SortTime"]))
             event_title = row["EventTitle"]
             event_location = row["Location"]
+
             if event_location.startswith("Room"):
                 event_location = (
                     f"11 South Union St, Montgomery, AL 36130. {event_location}"
@@ -89,8 +110,7 @@ def scrape(self, start=None):
                 event, {"11 south union": (32.37707594063977, -86.29919861850152)}
             )
 
-            bills = re.findall(r"(SB\s*\d+)", event_title, flags=re.IGNORECASE)
-            for bill in bills:
+            for bill in bills.get(row["OidMeeting"], []):
                 event.add_bill(bill)
 
             if row["AgendaUrl"]: