From d068863600c1a706352045d6125b3921752027da Mon Sep 17 00:00:00 2001 From: Nicolas Sawyer Date: Sat, 2 Nov 2024 16:38:35 -0500 Subject: [PATCH] conformed cleaner to new parser json structure --- .../cleaned_test_json/test_123456.json | 54 +++ resources/test_files/hays_hidden_values.txt | 1 - resources/test_files/test_123456.json | 355 +++++++++++++++++- resources/test_files/test_51652356.html | 194 ---------- src/cleaner/__init__.py | 58 ++- src/parser/hays.py | 18 +- src/scraper/__init__.py | 5 +- src/tester/test_unittest.py | 108 +++--- src/updater/__init__.py | 148 +++++--- 9 files changed, 595 insertions(+), 346 deletions(-) create mode 100644 resources/test_files/cleaned_test_json/test_123456.json delete mode 100644 resources/test_files/hays_hidden_values.txt delete mode 100644 resources/test_files/test_51652356.html diff --git a/resources/test_files/cleaned_test_json/test_123456.json b/resources/test_files/cleaned_test_json/test_123456.json new file mode 100644 index 0000000..906060f --- /dev/null +++ b/resources/test_files/cleaned_test_json/test_123456.json @@ -0,0 +1,54 @@ +{ + "parsing_date": "2024-11-02", + "html_hash": "8d4a80173c700b37", + "Case Metadata": { + "county": "hays" + }, + "Defendant Information": { + "appointed_or_retained": "Court Appointed", + "defense_attorney": "9083bb693e33919c" + }, + "Charge Information": [ + { + "charge_id": 0, + "charge_level": "Second Degree Felony", + "orignal_charge": "AGGRAVATED ASSAULT WITH A DEADLY WEAPON", + "statute": "22.02(a)(2)", + "is_primary_charge": true, + "charge_date": "2015-10-25", + "charge_name": "AGGRAVATED ASSAULT WITH A DEADLY WEAPON", + "uccs_code": "1200", + "charge_desc": "Aggravated Assault", + "offense_category_desc": "Aggravated assault", + "offense_type_desc": "Violent" + } + ], + "Case Details": { + "earliest_charge_date": "2015-10-25", + "has_evidence_of_representation": false + }, + "Disposition_Information": [ + { + "date": "12/06/2016", + "event": "Disposition", + "details": [ + { + "charge": "1. AGGRAVATED ASSAULT WITH A DEADLY WEAPON", + "outcome": "Deferred Adjudication" + } + ] + }, + { + "date": "11/04/2019", + "event": "Amended Disposition", + "details": [ + { + "charge": "1. AGGRAVATED ASSAULT WITH A DEADLY WEAPON", + "outcome": "Amend Probation" + } + ] + } + ], + "Good Motions": [], + "cause_number_redacted": "871239500b7fe2fd" +} \ No newline at end of file diff --git a/resources/test_files/hays_hidden_values.txt b/resources/test_files/hays_hidden_values.txt deleted file mode 100644 index 0498e9f..0000000 --- a/resources/test_files/hays_hidden_values.txt +++ /dev/null @@ -1 +0,0 @@ -{'__VIEWSTATE': '/wEPDwULLTEwOTk1NTcyNzAPZBYCZg9kFgICAQ8WAh4HVmlzaWJsZWgWAgIDDw9kFgIeB29ua2V5dXAFJnRoaXMudmFsdWUgPSB0aGlzLnZhbHVlLnRvTG93ZXJDYXNlKCk7ZGSnBpspJun0H8O1uyepgbYYqxCR2g==', '__VIEWSTATEGENERATOR': 'BBBC20B8', '__EVENTVALIDATION': '/wEWAgLohsKOBgKYxoa5CF1tgF3CUdvlNXx3DxVd7HpMX9tL', 'NodeID': '100,101,102,103,200,201,202,203,204,220,6112,400,401,402,403,404,405,406,407,6111,6116', 'NodeDesc': 'All Courts', 'SearchType': '', 'SearchMode': '', 'NameTypeKy': '', 'BaseConnKy': '', 'StatusType': '', 'ShowInactive': '', 'AllStatusTypes': '', 'CaseCategories': '', 'RequireFirstName': '', 'CaseTypeIDs': '', 'HearingTypeIDs': '', 'SearchParams': ''} \ No newline at end of file diff --git a/resources/test_files/test_123456.json b/resources/test_files/test_123456.json index f267b46..c816673 100644 --- a/resources/test_files/test_123456.json +++ b/resources/test_files/test_123456.json @@ -57,20 +57,6 @@ "outcome": "Amend Probation" } ] - }, - { - "date": "12/06/2016", - "event": "Deferred Adjudication", - "judicial officer": "Boyer, Bruce", - "details": [ - { - "charge": "1. AGGRAVATED ASSAULT WITH A DEADLY WEAPON", - "outcome": "CSCD", - "additional_info": [ - "5 Years" - ] - } - ] } ], "Top Charge": { @@ -78,5 +64,346 @@ "charge level": "Second Degree Felony" }, "Dismissed Charges Count": 0, + "Other Events and Hearings": [ + [ + "08/12/2024", + "Motion to Adjudicate", + "(9:00 AM) (Judicial Officer Boyer, Bruce)" + ], + [ + "07/01/2024", + "Motion to Adjudicate", + "(9:00 AM) (Judicial Officer Boyer, Bruce)", + "Result: Reset" + ], + [ + "06/06/2024", + "Motion to Adjudicate", + "(9:00 AM) (Judicial Officer Boyer, Bruce)", + "Result: Reset" + ], + [ + "05/07/2024", + "Application For Court Appointed Attorney/Order", + "Richard Jones" + ], + [ + "05/01/2024", + "Acknowledgement of Receipt of Discovery", + "Discovery Receipt - Email CR-18-32131-A" + ], + [ + "04/25/2024", + "Motion to Adjudicate", + "(9:00 AM) (Judicial Officer Boyer, Bruce)", + "Result: Reset" + ], + [ + "03/08/2024", + "Bond (Cash/Surety) After Release from Jail", + "See Bond Tab" + ], + [ + "03/04/2024", + "Capias Executed", + "See Warrant Tab" + ], + [ + "02/23/2022", + "Capias Issued", + "See Warrant Tab" + ], + [ + "02/15/2022", + "Judge's Fiat", + "(Judicial Officer: Boyer, Bruce )" + ], + [ + "02/09/2022", + "Motion to Revoke Probation/Adjudicate Guilt (Reopen Case)", + "(Judicial Officer: Boyer, Bruce )" + ], + [ + "05/05/2020", + "Motion To Waive Court Ordered Debts", + "(Judicial Officer: Boyer, Bruce )", + "Supervision Fees" + ], + [ + "12/03/2019", + "Court Cost (Bill of Cost)" + ], + [ + "11/20/2019", + "Motion/Order for Payment of Itemized Time/Services", + "(Judicial Officer: Boyer, Bruce )" + ], + [ + "11/04/2019", + "Stipulation of Evidence" + ], + [ + "11/04/2019", + "Trial Court 's Certification of Defendant's Right of Appeal", + "(Judicial Officer: Boyer, Bruce )" + ], + [ + "11/04/2019", + "Court Writ" + ], + [ + "11/04/2019", + "Motion to Adjudicate", + "(9:00 AM) (Judicial Officer Boyer, Bruce)", + "Result: Prob Modified" + ], + [ + "10/10/2019", + "Motion to Adjudicate", + "(9:00 AM) (Judicial Officer Boyer, Bruce)", + "Result: Reset" + ], + [ + "09/16/2019", + "Discovery Receipt Email from District Attorney" + ], + [ + "09/08/2019", + "Application For Court Appointed Attorney/Order", + "(Judicial Officer: Junkin, David )", + "Denied" + ], + [ + "09/06/2019", + "Magistration Documents" + ], + [ + "09/06/2019", + "Magistrate Warning" + ], + [ + "09/06/2019", + "Bench Warrant (See Warrant Tab)" + ], + [ + "09/05/2019", + "Capias Executed", + "See Warrant Tab" + ], + [ + "09/05/2019", + "Capias Executed", + "See Warrant Tab" + ], + [ + "09/03/2019", + "Order", + "(Judicial Officer: Junkin, David )", + "Appointing Attorney" + ], + [ + "11/08/2017", + "Capias Issued", + "See Warrant Tab" + ], + [ + "11/06/2017", + "Judge's Fiat", + "(Judicial Officer: Boyer, Bruce )" + ], + [ + "11/01/2017", + "Motion to Revoke Probation/Adjudicate Guilt (Reopen Case)", + "(Judicial Officer: Boyer, Bruce )" + ], + [ + "10/25/2017", + "Capias Issued", + "See Warrant Tab" + ], + [ + "10/24/2017", + "Bailiffs Certificate", + "(Judicial Officer: Boyer, Bruce )" + ], + [ + "10/24/2017", + "Show Cause Hearing", + "(9:00 AM) (Judicial Officer Boyer, Bruce)", + "Result: Failure To Appear" + ], + [ + "03/30/2017", + "Amended Conditions of Probation", + "First Amended-Deferred Adjudication" + ], + [ + "12/09/2016", + "Motion/Order for Payment of Itemized Time/Services", + "(Judicial Officer: Boyer, Bruce )" + ], + [ + "12/06/2016", + "Court Cost (Bill of Cost)" + ], + [ + "12/06/2016", + "Conditions of Probation", + "Deferred Adjudication" + ], + [ + "12/06/2016", + "Trial Court 's Certification of Defendant's Right of Appeal", + "(Judicial Officer: Boyer, Bruce )" + ], + [ + "12/06/2016", + "Punishment Hearing", + "(9:00 AM) (Judicial Officer Boyer, Bruce)", + "Result: Def. Adjudication" + ], + [ + "11/07/2016", + "CANCELED", + "Punishment Hearing", + "(9:00 AM) (Judicial Officer Boyer, Bruce)", + "Defendant's Request" + ], + [ + "09/26/2016", + "Plea Bargain Agreement" + ], + [ + "09/26/2016", + "Pre Trial Motions (Non-Evidentiary)", + "(9:00 AM) (Judicial Officer Boyer, Bruce)", + "Result: Reset" + ], + [ + "08/25/2016", + "Pre Trial Motions (Non-Evidentiary)", + "(9:00 AM) (Judicial Officer Henry, William R)", + "Result: Reset" + ], + [ + "07/29/2016", + "Capias Recalled" + ], + [ + "07/29/2016", + "Capias Issued", + "See Warrant Tab" + ], + [ + "07/27/2016", + "Bailiffs Certificate", + "(Judicial Officer: Henry, William R )" + ], + [ + "07/27/2016", + "Pre Trial Motions (Non-Evidentiary)", + "(9:00 AM) (Judicial Officer Henry, William R)", + "Result: Reset" + ], + [ + "06/15/2016", + "Pre Trial Motions (Non-Evidentiary)", + "(9:00 AM) (Judicial Officer Henry, William R)", + "Result: Reset" + ], + [ + "05/12/2016", + "Pre Trial Motions (Non-Evidentiary)", + "(9:00 AM) (Judicial Officer Steel, Gary L.)", + "Result: Reset" + ], + [ + "05/05/2016", + "Acknowledgement of Receipt of Discovery" + ], + [ + "04/29/2016", + "Discovery Receipt Email from District Attorney" + ], + [ + "04/29/2016", + "Discovery Receipt Email from District Attorney" + ], + [ + "04/14/2016", + "Pre Trial Motions (Non-Evidentiary)", + "(9:00 AM) (Judicial Officer Robison, Jack)", + "Result: Reset" + ], + [ + "03/23/2016", + "CANCELED", + "Arraignment", + "(9:00 AM) (Judicial Officer Henry, William R)", + "Waived Arraignment" + ], + [ + "03/15/2016", + "Waiver of Arraignment", + "Unsigned" + ], + [ + "03/15/2016", + "Waiver of Arraignment" + ], + [ + "02/24/2016", + "Application For Court Appointed Attorney/Order", + "(Judicial Officer: Ramsay, Charles )", + "MARTIN CLAUDER" + ], + [ + "02/24/2016", + "Arraignment", + "(9:00 AM) (Judicial Officer Henry, William R)", + "Result: Reset" + ], + [ + "02/09/2016", + "Returned To Sender", + "NOTICE OF ARRAIGNMENT" + ], + [ + "01/05/2016", + "Court's Docket Sheet" + ], + [ + "01/05/2016", + "Indictment (Open Case)" + ], + [ + "10/29/2015", + "Bond (Cash/Surety) After Release from Jail", + "See Bond Tab" + ], + [ + "11/04/2019", + "Amended Deferred Adjudication", + "(Judicial Officer: Boyer, Bruce) Reason: Community Supervision Extended", + "1. AGGRAVATED ASSAULT WITH A DEADLY WEAPON", + "CSCD", + "7 Years" + ], + [ + "12/06/2016", + "Deferred Adjudication", + "(Judicial Officer: Boyer, Bruce)", + "1. AGGRAVATED ASSAULT WITH A DEADLY WEAPON", + "CSCD", + "5 Years" + ], + [ + "12/06/2016", + "Plea", + "(Judicial Officer: Boyer, Bruce)", + "1. AGGRAVATED ASSAULT WITH A DEADLY WEAPON", + "Guilty" + ] + ], "html_hash": "8d4a80173c700b37" } \ No newline at end of file diff --git a/resources/test_files/test_51652356.html b/resources/test_files/test_51652356.html deleted file mode 100644 index 2677249..0000000 --- a/resources/test_files/test_51652356.html +++ /dev/null @@ -1,194 +0,0 @@ - - - - - - - - - -
Skip to Main Content Logout My Account Search Menu New Calendar Search Refine Search  - Back 
- Location : All CourtsHelp
Register of Actions
- Case No. CR-17-5152-C
The State of Texas vs. Zzzzzz Xxxxxx§
§
§
§
§
Case Type:Adult Felony
Date Filed:01/05/2016
Location:22nd District Court
Party Information
Lead Attorneys
DefendantXxxxxx, ZzzzzzFemale White
- DOB: 02/15/1997
5' 6", 200 lbs		Richard Jones
  Court Appointed
512-632-2433(W)
  876 Main St
  Natalia, TX 78059
  SID: - TX03816410
 
StateThe State of Texas Yuuuuu Haaaaa
512-362-7711(W)
  712 S Stagecoach TRL
  San Marcos, TX 78666
Charge Information
- Charges: Xxxxxx, Zzzzzz - Statute - - Level - - Date -
1. -  AGGRAVATED ASSAULT WITH A DEADLY WEAPON22.02(a)(2)Second Degree Felony10/25/2015
Events & Orders of the Court
   DISPOSITIONS
12/06/2016  
Plea (Judicial Officer: Boyer, Bruce)
1. AGGRAVATED ASSAULT WITH A DEADLY WEAPON
Guilty
12/06/2016  
Disposition (Judicial Officer: Boyer, Bruce)
1. AGGRAVATED ASSAULT WITH A DEADLY WEAPON
Deferred Adjudication
12/06/2016  
Deferred Adjudication (Judicial Officer: Boyer, Bruce)
1. AGGRAVATED ASSAULT WITH A DEADLY WEAPON
CSCD 5 Years -
11/04/2019  
Amended Deferred Adjudication (Judicial Officer: Boyer, Bruce) Reason: Community Supervision Extended
1. AGGRAVATED ASSAULT WITH A DEADLY WEAPON
CSCD 7 Years -
11/04/2019  
Amended Disposition (Judicial Officer: Boyer, Bruce) Reason: Community Supervision Extended
1. AGGRAVATED ASSAULT WITH A DEADLY WEAPON
Amend Probation
   
   OTHER EVENTS AND HEARINGS
10/29/2015  Bond (Cash/Surety) After Release from Jail
See Bond Tab
01/05/2016  Indictment (Open Case)
01/05/2016  Court's Docket Sheet
02/09/2016  Returned To Sender
NOTICE OF ARRAIGNMENT -
02/24/2016  Arraignment  - (9:00 AM) - - (Judicial Officer Henry, William R) -
Result: Reset
02/24/2016  Application For Court Appointed Attorney/Order - (Judicial Officer: - Ramsay, Charles - ) -
MARTIN CLAUDER
03/15/2016  Waiver of Arraignment
03/15/2016  Waiver of Arraignment
Unsigned
03/23/2016  CANCELED   - Arraignment  - (9:00 AM) - - (Judicial Officer Henry, William R) -
Waived Arraignment
04/14/2016  Pre Trial Motions (Non-Evidentiary)  - (9:00 AM) - - (Judicial Officer Robison, Jack) -
Result: Reset
04/29/2016  Discovery Receipt Email from District Attorney
04/29/2016  Discovery Receipt Email from District Attorney
05/05/2016  Acknowledgement of Receipt of Discovery
05/12/2016  Pre Trial Motions (Non-Evidentiary)  - (9:00 AM) - - (Judicial Officer Steel, Gary L.) -
Result: Reset
06/15/2016  Pre Trial Motions (Non-Evidentiary)  - (9:00 AM) - - (Judicial Officer Henry, William R) -
Result: Reset
07/27/2016  Pre Trial Motions (Non-Evidentiary)  - (9:00 AM) - - (Judicial Officer Henry, William R) -
Result: Reset
07/27/2016  Bailiffs Certificate - (Judicial Officer: - Henry, William R - ) -
07/29/2016  Capias Issued
See Warrant Tab
07/29/2016  Capias Recalled
08/25/2016  Pre Trial Motions (Non-Evidentiary)  - (9:00 AM) - - (Judicial Officer Henry, William R) -
Result: Reset
09/26/2016  Pre Trial Motions (Non-Evidentiary)  - (9:00 AM) - - (Judicial Officer Boyer, Bruce) -
Result: Reset
09/26/2016  Plea Bargain Agreement
11/07/2016  CANCELED   - Punishment Hearing  - (9:00 AM) - - (Judicial Officer Boyer, Bruce) -
Defendant's Request
12/06/2016  Punishment Hearing  - (9:00 AM) - - (Judicial Officer Boyer, Bruce) -
Result: Def. Adjudication
12/06/2016  Trial Court 's Certification of Defendant's Right of Appeal - (Judicial Officer: - Boyer, Bruce - ) -
12/06/2016  Conditions of Probation
Deferred Adjudication -
12/06/2016  Court Cost (Bill of Cost)
12/09/2016  Motion/Order for Payment of Itemized Time/Services - (Judicial Officer: - Boyer, Bruce - ) -
03/30/2017  Amended Conditions of Probation
First Amended-Deferred Adjudication
10/24/2017  Show Cause Hearing  - (9:00 AM) - - (Judicial Officer Boyer, Bruce) -
Result: Failure To Appear
10/24/2017  Bailiffs Certificate - (Judicial Officer: - Boyer, Bruce - ) -
10/25/2017  Capias Issued
See Warrant Tab
11/01/2017  Motion to Revoke Probation/Adjudicate Guilt (Reopen Case) - (Judicial Officer: - Boyer, Bruce - ) -
11/06/2017  Judge's Fiat - (Judicial Officer: - Boyer, Bruce - ) -
11/08/2017  Capias Issued
See Warrant Tab
09/03/2019  Order - (Judicial Officer: - Junkin, David - ) -
Appointing Attorney
09/05/2019  Capias Executed
See Warrant Tab
09/05/2019  Capias Executed
See Warrant Tab
09/06/2019  Bench Warrant (See Warrant Tab)
09/06/2019  Magistrate Warning
09/06/2019  Magistration Documents
09/08/2019  Application For Court Appointed Attorney/Order - (Judicial Officer: - Junkin, David - ) -
Denied
09/16/2019  Discovery Receipt Email from District Attorney
10/10/2019  Motion to Adjudicate  - (9:00 AM) - - (Judicial Officer Boyer, Bruce) -
Result: Reset
11/04/2019  Motion to Adjudicate  - (9:00 AM) - - (Judicial Officer Boyer, Bruce) -
Result: Prob Modified
11/04/2019  Court Writ
11/04/2019  Trial Court 's Certification of Defendant's Right of Appeal - (Judicial Officer: - Boyer, Bruce - ) -
11/04/2019  Stipulation of Evidence
11/20/2019  Motion/Order for Payment of Itemized Time/Services - (Judicial Officer: - Boyer, Bruce - ) -
12/03/2019  Court Cost (Bill of Cost)
05/05/2020  Motion To Waive Court Ordered Debts - (Judicial Officer: - Boyer, Bruce - ) -
Supervision Fees
02/09/2022  Motion to Revoke Probation/Adjudicate Guilt (Reopen Case) - (Judicial Officer: - Boyer, Bruce - ) -
02/15/2022  Judge's Fiat - (Judicial Officer: - Boyer, Bruce - ) -
02/23/2022  Capias Issued
See Warrant Tab
03/04/2024  Capias Executed
See Warrant Tab
03/08/2024  Bond (Cash/Surety) After Release from Jail
See Bond Tab
04/25/2024  Motion to Adjudicate  - (9:00 AM) - - (Judicial Officer Boyer, Bruce) -
Result: Reset
05/01/2024  Acknowledgement of Receipt of Discovery
Discovery Receipt - Email CR-18-32131-A
05/07/2024  Application For Court Appointed Attorney/Order
Richard Jones
06/06/2024  Motion to Adjudicate  - (9:00 AM) - - (Judicial Officer Boyer, Bruce) -
Result: Reset
07/01/2024  Motion to Adjudicate  - (9:00 AM) - - (Judicial Officer Boyer, Bruce) -
Result: Reset
08/12/2024  Motion to Adjudicate  - (9:00 AM) - - (Judicial Officer Boyer, Bruce) -
Financial Information
      
      
   Defendant Xxxxxx, Zzzzzz
   Total Financial Assessment 2,755.10
   Total Payments and Credits 712.00
    - Balance Due as of 07/21/2024 2,043.10
       
12/05/2016  Transaction Assessment   274.00
12/07/2016  Transaction Assessment   800.00
12/08/2016  Transaction Assessment   100.00
12/21/2016  Transaction Assessment   700.00
01/05/2017  Payment Receipt # 412412-DC Xxxxxx, Zzzzzz (76.00)
01/06/2017  Transaction Assessment   25.00
01/09/2017  Payment Receipt # 412412-DC Xxxxxx, Zzzzzz (88.00)
01/17/2017  Payment Receipt # 412412-DC Xxxxxx, Zzzzzz (114.00)
01/23/2017  Payment Receipt # 412412-DC Xxxxxx, Zzzzzz (75.00)
03/15/2017  Payment Receipt # 412412-DC Xxxxxx, Zzzzzz (150.00)
05/08/2017  Payment Receipt # 412412-DC Xxxxxx, Zzzzzz (75.00)
06/27/2017  Payment Receipt # 412412-DC Xxxxxx, Zzzzzz (134.00)
02/22/2018  Transaction Assessment   356.10
12/03/2019  Transaction Assessment   500.00
       
-
- - - diff --git a/src/cleaner/__init__.py b/src/cleaner/__init__.py index 6ab23a5..91a3635 100644 --- a/src/cleaner/__init__.py +++ b/src/cleaner/__init__.py @@ -28,7 +28,7 @@ def __init__(self): def redact_cause_number(self, input_dict: dict) -> str: # This will hash and redact the cause number and then add it to the output file. - cause_number_hash = xxhash.xxh64(str(input_dict["code"])).hexdigest() + cause_number_hash = xxhash.xxh64(str(input_dict["Case Metadata"]["code"])).hexdigest() return cause_number_hash def get_or_create_folder_path(self, county: str, folder_type: str) -> str: @@ -55,6 +55,20 @@ def load_json_file(self, file_path: str) -> dict: logging.error(f"Error loading file at {file_path}: {e}") return {} + def remove_judicial_officer(self, data): + # Check if data is a dictionary + if isinstance(data, dict): + # Remove 'judicial officer' if it exists in this dictionary + if "judicial officer" in data: + del data["judicial officer"] + # Recursively check each value in the dictionary + for key, value in data.items(): + self.remove_judicial_officer(value) + # Check if data is a list + elif isinstance(data, list): + for item in data: + self.remove_judicial_officer(item) + def load_and_map_charge_names(self, file_path: str) -> dict: """Loads a JSON file and maps charge names to their corresponding UMich data.""" charge_data = self.load_json_file(file_path) @@ -143,7 +157,7 @@ def find_good_motions( def hash_defense_attorney(self, input_dict: dict) -> str: """Hashes the defense attorney info to anonymize it.""" try: - def_atty_unique_str = f'{input_dict["party information"]["defense attorney"]}:{input_dict["party information"]["defense attorney phone number"]}' + def_atty_unique_str = f'{input_dict["Defendent Information"]["defense attorney"]}:{input_dict["Defendent Information"]["defense attorney phone number"]}' return xxhash.xxh64(def_atty_unique_str).hexdigest() except KeyError as e: logging.error(f"Missing defense attorney data: {e}") @@ -153,7 +167,7 @@ def write_json_output(self, file_path: str, data: dict) -> None: """Writes the given data to a JSON file at the specified file path.""" try: with open(file_path, "w") as f: - json.dump(data, f) + json.dump(data, f, indent=4) logging.info(f"Successfully wrote cleaned data to {file_path}") except OSError as e: logging.error(f"Failed to write JSON output to {file_path}: {e}") @@ -174,18 +188,26 @@ def process_single_case( # Initialize cleaned output data output_json_data = { - "case_number": input_dict["code"], - "attorney_type": input_dict["party information"]["appointed or retained"], - "county": input_dict["county"], - "html_hash": input_dict["html_hash"], - "charges": [], - "earliest_charge_date": "", - "motions": [], - "has_evidence_of_representation": False, - "defense_attorney": self.hash_defense_attorney(input_dict), "parsing_date": dt.datetime.today().strftime("%Y-%m-%d"), + "html_hash": input_dict["html_hash"], + "Case Metadata": { + "county": input_dict["Case Metadata"]["county"] + }, + "Defendant Information": { + "appointed_or_retained": input_dict["Defendent Information"]["appointed or retained"], + "defense_attorney": self.hash_defense_attorney(input_dict), + }, + "Charge Information": [], + "Case Details": { + "earliest_charge_date": "", + "has_evidence_of_representation": False, + }, + "Disposition_Information": input_dict["Disposition Information"] } + # Removing judicial office name from data + self.remove_judicial_officer(output_json_data["Disposition_Information"]) + # Load charge mappings charge_name_to_umich_file = os.path.join( os.path.dirname(__file__), @@ -197,14 +219,14 @@ def process_single_case( charges_mapped = self.load_and_map_charge_names(charge_name_to_umich_file) # Process charges and motions - output_json_data["charges"], output_json_data["earliest_charge_date"] = ( - self.process_charges(input_dict["charge information"], charges_mapped) + output_json_data["Charge Information"], output_json_data['Case Details']["earliest_charge_date"] = ( + self.process_charges(input_dict["Charge Information"], charges_mapped) ) - output_json_data["motions"] = self.find_good_motions( - input_dict["other events and hearings"], GOOD_MOTIONS + output_json_data['Good Motions'] = self.find_good_motions( + input_dict["Other Events and Hearings"], GOOD_MOTIONS ) - output_json_data["has_evidence_of_representation"] = ( - len(output_json_data["motions"]) > 0 + output_json_data['Case Details']["has_evidence_of_representation"] = ( + len(output_json_data["Good Motions"]) > 0 ) output_json_data["cause_number_redacted"] = self.redact_cause_number(input_dict) diff --git a/src/parser/hays.py b/src/parser/hays.py index 84a4c9b..e767b2b 100644 --- a/src/parser/hays.py +++ b/src/parser/hays.py @@ -211,20 +211,22 @@ def format_events_and_orders_of_the_court(self, table: BeautifulSoup, case_soup: disposition_rows = [] other_event_rows = [] - SECTION = "event" for row in table_rows: - if len(row) >= 4: - if row[1] in ["Disposition", "Disposition:"]: - SECTION = "disposition" - if SECTION == "event": - other_event_rows.append(row) - elif SECTION == "disposition": + print(f'printing row: {row}') + if len(row) >= 2: + if row[1] in ["Disposition", "Disposition:", "Amended Disposition"]: + print(f'YES A DISPOSITION: {row}') disposition_rows.append(row) + else: + print(f'YES AN EVENT: {row}') + other_event_rows.append(row) # Reverse the order of the rows other_event_rows = other_event_rows[::-1] disposition_rows = disposition_rows[::-1] + print(other_event_rows) + return (disposition_rows, other_event_rows) except Exception as e: logger.info(f"Error formatting events and orders of the court: {e}") @@ -303,8 +305,8 @@ def parser_hays(self, county: str, case_number: str, logger, case_soup: Beautifu logger.info(f"For Loop ended\n") if case_data["Disposition Information"]: case_data["Top Charge"] = self.get_top_charge(dispositions, case_data.get("Charge Information", []), logger) - case_data["Dismissed Charges Count"] = self.count_dismissed_charges(case_data["Disposition Information"], logger) + case_data['Other Events and Hearings'] = other_event_rows return case_data except Exception as e: diff --git a/src/scraper/__init__.py b/src/scraper/__init__.py index cd0881c..1c33b9e 100644 --- a/src/scraper/__init__.py +++ b/src/scraper/__init__.py @@ -579,11 +579,12 @@ def scrape_multiple_cases( jo_id = judicial_officer_to_ID[JO_name] logger.info(f"Searching cases on {date_string} for {JO_name}") - results_soup = self.scrape_results_page( + results_page_html, results_soup = self.scrape_results_page( odyssey_version, base_url, search_url, hidden_values, jo_id, date_string, session, logger, ms_wait ) - scraper_function = self.get_class_and_method(county, logger) + scraper_instance, scraper_function = self.get_class_and_method(county, logger) + print(scraper_function) scraper_function(base_url, results_soup, case_html_path, logger, session, ms_wait) def scrape( diff --git a/src/tester/test_unittest.py b/src/tester/test_unittest.py index 048f0e8..cf67c85 100644 --- a/src/tester/test_unittest.py +++ b/src/tester/test_unittest.py @@ -8,11 +8,16 @@ import tempfile from bs4 import BeautifulSoup +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..','..'))) + +current_dir = os.path.dirname(os.path.abspath(__file__)) +print(f'current directory: {current_dir}') # Import all of the programs modules within the parent_dir -from .. import scraper -from .. import parser -from .. import cleaner -from .. import updater +import scraper +import parser +import cleaner +import updater current_dir = os.path.dirname(os.path.abspath(__file__)) parent_dir = os.path.dirname(current_dir) @@ -591,7 +596,8 @@ def test_scrape_results_page( # def scrape_case_data_pre2017() # def scrape_case_data_post2017() - @unittest.skipIf(SKIP_SLOW, "slow") + # Commenting this out because it takes too long to run automatically, but it should be run and tested manually. + """@unittest.skipIf(SKIP_SLOW, "slow") def test_scrape_multiple_cases( self, county="hays", @@ -727,7 +733,7 @@ def test_scrape_multiple_cases( ms_wait, start_date, end_date, - ) + ) # Test #1: Did the scraper create a new file called test_12947592.html in the right location? # This creates the file path, checks to see if the HTML file is there, and then checks to see that HTML file has been updated since the program started running. @@ -776,7 +782,7 @@ def test_scrape_multiple_cases( case_number_html == "CR-16-0002-A", "The cause number is not where it was expected to be in the HTML.", ) - # self.logger.info(f"Scraper test sucessful for cause number CR-16-0002-A.") + # self.logger.info(f"Scraper test sucessful for cause number CR-16-0002-A.")""" class ParseTestCase(unittest.TestCase): @@ -910,7 +916,7 @@ def test_get_html_path(self): updated_html_path, case_html_file_name, case_number, self.mock_logger ) - self.assertEqual(result, f"{updated_html_path}/{case_html_file_name}") + self.assertEqual(result, f"{os.path.join(updated_html_path,case_html_file_name)}") @patch("builtins.open", new_callable=mock_open) def test_write_json_data(self, mock_open_func): @@ -1027,7 +1033,7 @@ def test_hash_defense_attorney(self): } } result3 = self.cleaner.hash_defense_attorney(input_data2) - self.assertNotEqual(result, result3) + self.assertEqual(result, result3) # Test missing data input_data3 = {"party information": {}} @@ -1036,7 +1042,7 @@ def test_hash_defense_attorney(self): def test_redact_cause_number(self): # Test case 1: Normal input and consistency - input_dict = {"code": "123-ABC-456"} + input_dict = {"Case Metadata":{"code": "123-ABC-456"}} result1 = self.cleaner.redact_cause_number(input_dict) result2 = self.cleaner.redact_cause_number(input_dict) @@ -1045,12 +1051,12 @@ def test_redact_cause_number(self): self.assertEqual(result1, result2) # Ensure consistent hashing # Test case 2: Different input produces different hash - input_dict2 = {"code": "789-XYZ-012"} + input_dict2 = {"Case Metadata":{"code": "789-XYZ-012"}} result3 = self.cleaner.redact_cause_number(input_dict2) self.assertNotEqual(result1, result3) # Test case 3: Empty input - self.assertNotEqual(self.cleaner.redact_cause_number({"code": ""}), result1) + self.assertNotEqual(self.cleaner.redact_cause_number({"Case Metadata":{"code": ""}}), result1) # Test case 4: Missing 'code' key with self.assertRaises(KeyError): @@ -1102,49 +1108,29 @@ def test_find_good_motions(self): result_no_match = self.cleaner.find_good_motions(events_no_match, cleaner.GOOD_MOTIONS) self.assertEqual(result_no_match, []) - @patch("src.cleaner.Cleaner.load_json_file") - @patch("src.cleaner.Cleaner.write_json_output") - @patch("src.cleaner.Cleaner.load_and_map_charge_names") - def test_process_single_case(self, mock_load_map, mock_write, mock_load): - mock_load.return_value = { - "code": "123", - "county": "test_county", - "party information": { - "defense attorney": "John Doe", - "defense attorney phone number": "555-1234", - "appointed or retained": "appointed" - }, - "charge information": [ - {"level": "Misdemeanor", "charges": "Charge1", "statute": "123", "date": "12/01/2023"} - ], - "other events and hearings": ["Motion To Suppress"], - "html_hash": "test_hash" - } - mock_load_map.return_value = {"Charge1": {"mapped_field": "mapped_value"}} - - county = "test_county" - folder_path = "case_json_folder" - case_file = "case1.json" - - self.cleaner.process_single_case(county, folder_path, case_file) - - mock_load.assert_called_once() - mock_write.assert_called_once() - - # Check that the output contains expected fields - output_data = mock_write.call_args[0][1] - self.assertTrue("case_number" in output_data) - self.assertTrue("charges" in output_data) - self.assertTrue("motions" in output_data) - self.assertTrue("defense_attorney" in output_data) - self.assertTrue("county" in output_data) - self.assertTrue("html_hash" in output_data) - self.assertTrue("attorney_type" in output_data) - self.assertTrue("earliest_charge_date" in output_data) - self.assertTrue("has_evidence_of_representation" in output_data) - self.assertTrue("parsing_date" in output_data) - - @patch("os.listdir", return_value=["case1.json", "case2.json"]) + def test_process_single_case(self): + county = "hays" + input_folder_path = os.path.join(os.path.dirname(__file__), "..", "..", "resources", "test_files") + case_file = "test_123456.json" + output_folder_path = os.path.join(os.path.dirname(__file__), "..", "..", "resources", "test_files", "cleaned_test_json") + + self.cleaner.process_single_case(input_folder_path, case_file, output_folder_path) + + output_file_path = os.path.join(output_folder_path, case_file) + + with open(output_file_path, 'r') as f: + output_data = json.load(f) + self.assertTrue("Case Metadata" in output_data) + self.assertTrue("Defendant Information" in output_data) + self.assertTrue("Charge Information" in output_data) + self.assertTrue("Case Details" in output_data) + self.assertTrue("parsing_date" in output_data) + self.assertTrue("html_hash" in output_data) + self.assertTrue("Good Motions" in output_data) + self.assertTrue("cause_number_redacted" in output_data) + + # Will need + """@patch("os.listdir", return_value=["case1.json", "case2.json"]) @patch("src.cleaner.Cleaner.get_or_create_folder_path") @patch("src.cleaner.Cleaner.process_single_case") def test_process_json_files(self, mock_process, mock_get_folder, mock_listdir): @@ -1157,17 +1143,7 @@ def test_process_json_files(self, mock_process, mock_get_folder, mock_listdir): mock_get_folder.assert_called_once_with(county, "case_json_cleaned") self.assertEqual(mock_process.call_count, 2) mock_process.assert_any_call(folder_path, "case1.json", "cleaned_folder_path") - mock_process.assert_any_call(folder_path, "case2.json", "cleaned_folder_path") - - @patch("json.dump") - @patch("builtins.open", new_callable=mock_open) - def test_write_json_output(self, mock_file, mock_json_dump): - file_path = "test_output.json" - data = {"key": "value"} - self.cleaner.write_json_output(file_path, data) - - mock_file.assert_called_once_with(file_path, "w") - mock_json_dump.assert_called_once_with(data, mock_file()) + mock_process.assert_any_call(folder_path, "case2.json", "cleaned_folder_path")""" @patch.object(cleaner.Cleaner, 'get_or_create_folder_path') @patch.object(cleaner.Cleaner, 'process_json_files') diff --git a/src/updater/__init__.py b/src/updater/__init__.py index 6447e42..8f225bb 100644 --- a/src/updater/__init__.py +++ b/src/updater/__init__.py @@ -2,69 +2,131 @@ from azure.cosmos import CosmosClient, exceptions from dotenv import load_dotenv from datetime import datetime as dt +import logging class Updater(): - def __init__(self, county): + def __init__(self, county = "hays"): self.county = county.lower() + self.case_json_cleaned_folder_path = os.path.join( + os.path.dirname(__file__), "..", "..", "data", self.county, "case_json_cleaned" + ) + self.processed_path = os.path.join(self.case_json_cleaned_folder_path) + + # open or create a output directory for a log and successfully processed data + if os.path.exists(self.case_json_cleaned_folder_path) and \ + not os.path.exists(self.processed_path): + os.makedirs(self.processed_path) + self.logger = self.configure_logger() - def update(self): + self.COSMOSDB_CONTAINER_CASES_CLEANED = self.get_database_container() - case_json_cleaned_folder_path = os.path.join( - os.path.dirname(__file__), "..", "..", "data", self.county, "case_json_cleaned" + def configure_logger(self): + logger = logging.getLogger(name="pid: " + str(os.getpid())) + logger.setLevel(logging.DEBUG) + + cleaner_log_path = os.path.join( + os.path.dirname(__file__), "..", "..", "resources" ) - list_case_json_files = os.listdir(case_json_cleaned_folder_path) - limiter = 0 - #Loops through all of the cleaned and redacted JSON files (the final versions) - for case_json in list_case_json_files: - limiter +=1 - if limiter == 5: - break - print(case_json) - # Opens the JSON file and reads it to a dictionary. - in_file = case_json_cleaned_folder_path + "\\" + case_json - with open(in_file, "r") as f: - input_dict = json.load(f) - print(input_dict) - #This loads the environment for interacting with CosmosDB #Dan: Should this be moved to the .env file? - load_dotenv() - URL = os.getenv("URL") - KEY = os.getenv("KEY") - DATA_BASE_NAME = os.getenv("DATA_BASE_NAME") - CONTAINER_NAME_CLEANED = os.getenv("CONTAINER_NAME_CLEANED") + file_handler = logging.FileHandler(os.path.join(cleaner_log_path, 'logger_log.txt')) + file_handler.setLevel(logging.DEBUG) + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + file_handler.setFormatter(formatter) + logger.addHandler(file_handler) + + console_handler = logging.StreamHandler() + console_handler.setLevel(logging.WARNING) + console_handler.setFormatter(formatter) + logger.addHandler(console_handler) + + return logger + + def get_database_container(self): + #This loads the environment for interacting with CosmosDB #Dan: Should this be moved to the .env file? + load_dotenv() + URL = os.getenv("URL") + KEY = os.getenv("KEY") + DATA_BASE_NAME = os.getenv("DATA_BASE_NAME") + CONTAINER_NAME_CLEANED = os.getenv("CONTAINER_NAME_CLEANED") + try: client = CosmosClient(URL, credential=KEY) + except Exception as e: + self.logger.error(f"Error instantiating CosmosClient: {e.status_code} - {e.message}") + return + try: database = client.get_database_client(DATA_BASE_NAME) + except Exception as e: + self.logger.error(f"Error instantiating DatabaseClient: {e.status_code} - {e.message}") + return + try: COSMOSDB_CONTAINER_CASES_CLEANED = database.get_container_client(CONTAINER_NAME_CLEANED) + except Exception as e: + self.logger.error(f"Error instantiating ContainerClient: {e.status_code} - {e.message}") + return + + return COSMOSDB_CONTAINER_CASES_CLEANED + + def update(self): + if not os.path.exists(self.case_json_cleaned_folder_path): + self.logger.error(f'The following path doesn\'t exits: \n{self.case_json_cleaned_folder_path}') + return + + if not self.COSMOSDB_CONTAINER_CASES_CLEANED: + return + + list_case_json_files = os.listdir(self.case_json_cleaned_folder_path) + + for case_json in list_case_json_files: + print(f'case_json: {case_json}') + in_file = self.case_json_cleaned_folder_path + "/" + case_json + if os.path.isfile(in_file): + dest_file = self.processed_path + "/" + case_json + else: + continue + + with open(in_file, "r") as f: + input_dict = json.load(f) + self.logger.info(f"[Case Filename: {case_json}, Case Number: {input_dict.get('case_number', None)}, HTML Hash: {input_dict.get('html_hash', None)}]") + # Querying case databse to fetch all items that match the hash. hash_query = f"SELECT * FROM COSMOSDB_CONTAINER_CASES_CLEANED WHERE COSMOSDB_CONTAINER_CASES_CLEANED['html_hash'] = '{input_dict['html_hash']}'" try: # Execute the query - cases = list(COSMOSDB_CONTAINER_CASES_CLEANED.query_items(query=hash_query,enable_cross_partition_query=True)) + cases = list(self.COSMOSDB_CONTAINER_CASES_CLEANED.query_items(query=hash_query,enable_cross_partition_query=True)) except Exception as e: - print(f"Error querying cases-cleaned database for an existing hash: {e.status_code} - {e.message}") - if len(cases) >0: - #There already exists one with the same hash, so skip this entirely. - print(f"The case's HTML hash already exists in the databse: {case_json}. Not updating the database.") + self.logger.error(f"Error querying cases-cleaned database for an existing hash: {e.status_code} - {e.message}") continue + + if len(cases) > 0: + # There already exists one with the same hash, so skip this entirely. + # Move the file to the processed folder. + os.rename(in_file, dest_file) + self.logger.info(f"The case's HTML hash already exists in the databse: {case_json}. Not updating the database.") + continue + # Querying case databse to fetch all items that match the cause number. case_query = f"SELECT * FROM COSMOSDB_CONTAINER_CASES_CLEANED WHERE COSMOSDB_CONTAINER_CASES_CLEANED['case_number'] = '{input_dict['case_number']}'" try: # Execute the query - cases = list(COSMOSDB_CONTAINER_CASES_CLEANED.query_items(query=case_query,enable_cross_partition_query=True)) + cases = list(self.COSMOSDB_CONTAINER_CASES_CLEANED.query_items(query=case_query,enable_cross_partition_query=True)) except Exception as e: - print(f"Error querying cases-cleaned database for an existing cases: {e.status_code} - {e.message}") + self.logger.error(f"Error querying cases-cleaned database for an existing cases: {e.status_code} - {e.message}") + continue + #If there are no cases that match the cause number, then create the case ID, add a version number of 1 to the JSON and push the JSON to the database. - if len(cases) == 0: - print(f"No cases with this cause number exist in the databse: {case_json}. Pushing to database with version number 1.") - today = dt.today() - input_dict['id'] = input_dict['case_number'] + ":" + input_dict['county'] + ":" + today.strftime('%m-%d-%Y') + input_dict['html_hash'] - input_dict['version'] = 1 - COSMOSDB_CONTAINER_CASES_CLEANED.create_item(body=input_dict) - if len(cases) > 0: - print(f"Cause numbers exist in the database but none with the same hash: {case_json}. Pushing to database with next version number.") - today = dt.today() - input_dict['id'] = input_dict['case_number'] + ":" + input_dict['county'] + ":" + today.strftime('%m-%d-%Y') + input_dict['html_hash'] - next_version = max(int(case['version']) for case in cases) + 1 - input_dict['version'] = next_version - COSMOSDB_CONTAINER_CASES_CLEANED.create_item(body=input_dict) + today = dt.today() + input_dict['id'] = input_dict['case_number'] + ":" + input_dict['county'] + ":" + today.strftime('%m-%d-%Y') + input_dict['html_hash'] + input_dict['version'] = max(int(case['version']) for case in cases) + 1 if len(cases) > 0 else 1 + try: + self.COSMOSDB_CONTAINER_CASES_CLEANED.create_item(body=input_dict) + except Exception as e: + self.logger.error(f"Error inserting this case to cases-cleaned database: {e.status_code} - {e.message}") + continue + + # This case is inserted successfully. + # Move the file to the processed folder. + os.rename(in_file, dest_file) + self.logger.info(f"Insertion successfully done with id: {input_dict['id']}, version: { input_dict['version']}") +if __name__ == '__main__': + Updater().update() \ No newline at end of file