Skip to content

Commit

Permalink
Merge pull request #4862 from NewAgeAirbender/ca_bills
Browse files Browse the repository at this point in the history
CA: add dedupe keys to events_web chamber scrapes
  • Loading branch information
NewAgeAirbender authored Mar 5, 2024
2 parents 494a68e + 664e065 commit 3e8f088
Showing 1 changed file with 4 additions and 0 deletions.
4 changes: 4 additions & 0 deletions scrapers/ca/events_web.py
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,8 @@ def scrape_upper(self, start, end):
view_agenda_id = committee_row.xpath(
'.//button[contains(@class, "view-agenda")]/@data-nid'
)[0]
event_key = f"{hearing_title}#{when}#{view_agenda_id}"
event.dedupe_key = event_key
view_agenda_url = f"https://www.senate.ca.gov/getagenda?dfid={view_agenda_id}&type=committee"
self.scrape_upper_agenda(event, committees, view_agenda_url)
yield event
Expand Down Expand Up @@ -236,6 +238,8 @@ def scrape_lower(self):
start_date=when,
classification="committee-meeting",
)
event_key = f"{hearing_title}#{when}#{hearing_location}"
event.dedupe_key = event_key

committees = [
f"Assembly {com.strip()} Committee"
Expand Down

0 comments on commit 3e8f088

Please sign in to comment.