Merge pull request #3295 from catalyst-cooperative/phmsa-clean-up

Clean up and standardize PHMSA raw assets
catalyst-cooperative · Jan 26, 2024 · 1a2badd · 1a2badd
2 parents 219f4f8 + c0d6713
commit 1a2badd
Show file tree

Hide file tree

Showing 18 changed files with 299 additions and 294 deletions.
diff --git a/src/pudl/extract/phmsagas.py b/src/pudl/extract/phmsagas.py
@@ -59,19 +59,19 @@ def process_renamed(self, newdata: pd.DataFrame, page: str, **partition):
 raw_table_names = (
     "raw_phmsagas__yearly_distribution",
     "raw_phmsagas__yearly_transmission_gathering_summary_by_commodity",
-    "raw_phmsagas__yearly_miles_of_gathering_pipe_by_nps",
-    "raw_phmsagas__yearly_miles_of_transmission_pipe_by_nps",
-    "raw_phmsagas__yearly_inspections_and_assessments",
-    "raw_phmsagas__yearly_miles_of_pipe_by_class_location",
+    "raw_phmsagas__yearly_gathering_pipe_miles_by_nps",
+    "raw_phmsagas__yearly_transmission_pipe_miles_by_nps",
+    "raw_phmsagas__yearly_transmission_gathering_inspections_assessments",
+    "raw_phmsagas__yearly_transmission_gathering_pipe_miles_by_class_location",
     "raw_phmsagas__yearly_transmission_material_verification",
     "raw_phmsagas__yearly_transmission_hca_miles_by_determination_method_and_risk_model",
-    "raw_phmsagas__yearly_gas_transmission_miles_by_pressure_test_range_and_internal_inspection",
-    "raw_phmsagas__yearly_transmission_preparer_certification",
-    "raw_phmsagas__yearly_miles_of_transmission_pipe_by_smys",
-    "raw_phmsagas__yearly_transmission_failures_leaks_repairs",
+    "raw_phmsagas__yearly_transmission_miles_by_pressure_test_range_and_internal_inspection",
+    "raw_phmsagas__yearly_transmission_gathering_preparer_certification",
+    "raw_phmsagas__yearly_transmission_pipe_miles_by_smys",
+    "raw_phmsagas__yearly_transmission_gathering_failures_leaks_repairs",
     "raw_phmsagas__yearly_transmission_miles_by_maop",
-    "raw_phmsagas__yearly_miles_of_pipe_by_decade_installed",
-    "raw_phmsagas__yearly_miles_of_pipe_in_state_by_material",
+    "raw_phmsagas__yearly_transmission_gathering_pipe_miles_by_decade_installed",
+    "raw_phmsagas__yearly_transmission_gathering_pipe_miles_by_material",
 )
 
 phmsagas_raw_dfs = excel.raw_df_factory(Extractor, name="phmsagas")

diff --git a/...yearly_miles_of_gathering_pipe_by_nps.csv → ...ps/yearly_gathering_pipe_miles_by_nps.csv b/...yearly_miles_of_gathering_pipe_by_nps.csv → ...ps/yearly_gathering_pipe_miles_by_nps.csv
diff --git a/src/pudl/package_data/phmsagas/column_maps/yearly_miles_of_pipe_in_state_by_material.csv b/src/pudl/package_data/phmsagas/column_maps/yearly_miles_of_pipe_in_state_by_material.csv
diff --git a/...y_transmission_failures_leaks_repairs.csv → ...sion_gathering_failures_leaks_repairs.csv b/...y_transmission_failures_leaks_repairs.csv → ...sion_gathering_failures_leaks_repairs.csv
diff --git a/...ps/yearly_inspections_and_assessments.csv → ...ion_gathering_inspections_assessments.csv b/...ps/yearly_inspections_and_assessments.csv → ...ion_gathering_inspections_assessments.csv
diff --git a/...early_miles_of_pipe_by_class_location.csv → ...athering_pipe_miles_by_class_location.csv b/...early_miles_of_pipe_by_class_location.csv → ...athering_pipe_miles_by_class_location.csv
@@ -60,7 +60,3 @@ total_pipe_class_2_miles,,,,,,,,,,,,b4total_2,b4total_2,b4total_2,b4total_2,b4to
 total_pipe_class_3_miles,,,,,,,,,,,,b4total_3,b4total_3,b4total_3,b4total_3,b4total_3,b4total_3,b4total_3,b4total_3,b4total_3,partlc3tot,partlc3tot,partlc3tot,partlc3tot,partlc3tot,partlc3tot,partlc3tot,partlc3tot,partlc3tot,partlc3tot,partlc3tot,partlc3tot,partlc3tot
 total_pipe_class_4_miles,,,,,,,,,,,,b4total_4,b4total_4,b4total_4,b4total_4,b4total_4,b4total_4,b4total_4,b4total_4,b4total_4,partlc4tot,partlc4tot,partlc4tot,partlc4tot,partlc4tot,partlc4tot,partlc4tot,partlc4tot,partlc4tot,partlc4tot,partlc4tot,partlc4tot,partlc4tot
 total_pipe_class_total_miles,,,,,,,,,,,,b4total_t,b4total_t,b4total_t,b4total_t,b4total_t,b4total_t,b4total_t,b4total_t,b4total_t,partltotal,partltotal,partltotal,partltotal,partltotal,partltotal,partltotal,partltotal,partltotal,partltotal,partltotal,partltotal,partltotal
-total_pipe_hca_miles,,,,,,,,,,,,,,,,,,,,,partltrhcatotal,partltrhcatotal,partltrhcatotal,partltrhcatotal,partltrhcatotal,partltrhcatotal,partltrhcatotal,partltrhcatotal,partltrhcatotal,partltrhcatotal,partltrhcatotal,partltrhcatotal,partltrhcatotal
-total_pipe_192_710_miles,,,,,,,,,,,,,,,,,,,,,partltr192milestotal,partltr192milestotal,partltr192milestotal,partltr192milestotal,partltr192milestotal,partltr192milestotal,partltr192milestotal,partltr192milestotal,partltr192milestotal,partltr192milestotal,partltr192milestotal,partltr192milestotal,partltr192milestotal
-total_pipe_class_3_or_4_only_miles,,,,,,,,,,,,,,,,,,,,,partltrclass34total,partltrclass34total,partltrclass34total,partltrclass34total,partltrclass34total,partltrclass34total,partltrclass34total,partltrclass34total,partltrclass34total,partltrclass34total,partltrclass34total,partltrclass34total,partltrclass34total
-total_pipe_class_1_or_2_only_miles,,,,,,,,,,,,,,,,,,,,,partltrclass12total,partltrclass12total,partltrclass12total,partltrclass12total,partltrclass12total,partltrclass12total,partltrclass12total,partltrclass12total,partltrclass12total,partltrclass12total,partltrclass12total,partltrclass12total,partltrclass12total
diff --git a/...rly_miles_of_pipe_by_decade_installed.csv → ...hering_pipe_miles_by_decade_installed.csv b/...rly_miles_of_pipe_by_decade_installed.csv → ...hering_pipe_miles_by_decade_installed.csv
diff --git a/...ackage_data/phmsagas/column_maps/yearly_transmission_gathering_pipe_miles_by_material.csv b/...ackage_data/phmsagas/column_maps/yearly_transmission_gathering_pipe_miles_by_material.csv
diff --git a/...y_transmission_preparer_certification.csv → ...sion_gathering_preparer_certification.csv b/...y_transmission_preparer_certification.csv → ...sion_gathering_preparer_certification.csv
@@ -4,7 +4,9 @@ report_year,yr,yr,yr,yr,yr,yr,yr,yr,yr,yr,yr,yr,yr,yr,yr,yr,yr,yr,yr,yr,report_y
 report_number,rptid,rptid,rptid,rptid,rptid,rptid,rptid,rptid,rptid,rptid,rptid,rptid,rptid,rptid,rptid,rptid,rptid,rptid,rptid,rptid,report_number,report_number,report_number,report_number,report_number,report_number,report_number,report_number,report_number,report_number,report_number,report_number,report_number,report_number
 supplemental_report_number,,,,,,,,,,,,tr_sup_id,tr_sup_id,tr_sup_id,tr_sup_id,tr_sup_id,tr_sup_id,tr_sup_id,tr_sup_id,tr_sup_id,supplemental_number,supplemental_number,supplemental_number,supplemental_number,supplemental_number,supplemental_number,supplemental_number,supplemental_number,supplemental_number,supplemental_number,supplemental_number,supplemental_number,supplemental_number,supplemental_number
 operator_id_phmsa,opid,opid,opid,opid,opid,opid,opid,opid,opid,opid,opid,operator_id,operator_id,operator_id,operator_id,operator_id,operator_id,operator_id,operator_id,operator_id,operator_id,operator_id,operator_id,operator_id,operator_id,operator_id,operator_id,operator_id,operator_id,operator_id,operator_id,operator_id,operator_id,operator_id
+operator_parent_name_phmsa,,,,,,,,,,,,,,,,,,,,,parta2nameofparent_com,parta2nameofparent_com,,,,,,,,,,,,
 operator_name_phmsa,name,name,name,name,name,name,name,name,name,name,name,name,name,name,name,name,name,name,name,name,parta2nameofcomp,parta2nameofcomp,parta2nameofcomp,parta2nameofcomp,parta2nameofcomp,parta2nameofcomp,parta2nameofcomp,parta2nameofcomp,parta2nameofcomp,parta2nameofcomp,parta2nameofcomp,parta2nameofcomp,parta2nameofcomp,parta2nameofcomp
+operator_name_phmsa_alt,,,,,,,,,,,,,,,,,,,,,parta4nameofcomp,parta4nameofcomp,,,,,,,,,,,,
 headquarters_address_street,hstreet,hstreet,hstreet,hstreet,hstreet,haddr,haddr,haddr,haddr,hstreet,hstreet,hqstreet,hqstreet,hqstreet,hqstreet,hqstreet,hqstreet,hqstreet,hqstreet,hqstreet,parta4street,parta4street,parta4street,parta4street,parta4street,parta4street,parta4street,parta4street,parta4street,parta4street,parta4street,parta4street,parta4street,parta4street
 headquarters_address_city,hcity,hcity,hcity,hcity,hcity,hcity,hcity,hcity,hcity,hcity,hcity,hqcity,hqcity,hqcity,hqcity,hqcity,hqcity,hqcity,hqcity,hqcity,parta4city,parta4city,parta4city,parta4city,parta4city,parta4city,parta4city,parta4city,parta4city,parta4city,parta4city,parta4city,parta4city,parta4city
 headquarters_address_county,hcounty,hcounty,hcounty,hcounty,hcounty,hcnty,hcnty,hcnty,hcnty,hcounty,hcounty,hqcounty,hqcounty,hqcounty,hqcounty,hqcounty,hqcounty,hqcounty,hqcounty,hqcounty,,,,,,,,,,,,,,