From f825c27dbf4beeb9c8657a8f97c43cf5879992ad Mon Sep 17 00:00:00 2001 From: Austin Raney Date: Mon, 29 Jul 2024 21:20:23 -0400 Subject: [PATCH 1/3] refactor: split gpkg and legacy hf support into methods --- python/ngen_cal/src/ngen/cal/ngen.py | 72 ++++++++++++++++------------ 1 file changed, 41 insertions(+), 31 deletions(-) diff --git a/python/ngen_cal/src/ngen/cal/ngen.py b/python/ngen_cal/src/ngen/cal/ngen.py index 8e6dcb52..f8ae167f 100644 --- a/python/ngen_cal/src/ngen/cal/ngen.py +++ b/python/ngen_cal/src/ngen/cal/ngen.py @@ -115,45 +115,55 @@ def __init__(self, **kwargs): #Make a copy of the config file, just in case shutil.copy(self.realization, str(self.realization)+'_original') - #Read the catchment hydrofabric data + # Read the catchment hydrofabric data if self.hydrofabric is not None: - #Reading hydofabric from geopackage - self._catchment_hydro_fabric = gpd.read_file(self.hydrofabric, layer='divides') - self._catchment_hydro_fabric.set_index('divide_id', inplace=True) - self._nexus_hydro_fabric = gpd.read_file(self.hydrofabric, layer='nexus') - self._nexus_hydro_fabric.set_index('id', inplace=True) - self._flowpath_hydro_fabric = gpd.read_file(self.hydrofabric, layer='flowpaths') - self._flowpath_hydro_fabric.set_index('id', inplace=True) - attributes = gpd.read_file(self.hydrofabric, layer="flowpath_attributes").set_index('id') - self._x_walk = pd.Series( attributes[ ~ attributes['rl_gages'].isna() ]['rl_gages'] ) + self._read_hydrofabric() else: - # Legacy geojson support - assert self.catchments is not None, "missing geojson catchments file" - assert self.nexus is not None, "missing geojson nexus file" - assert self.crosswalk is not None, "missing crosswalk file" - self._catchment_hydro_fabric = gpd.read_file(self.catchments) - self._catchment_hydro_fabric = self._catchment_hydro_fabric.rename(columns=str.lower) - self._catchment_hydro_fabric.set_index('id', inplace=True) - self._nexus_hydro_fabric = gpd.read_file(self.nexus) - self._nexus_hydro_fabric = self._nexus_hydro_fabric.rename(columns=str.lower) - self._nexus_hydro_fabric.set_index('id', inplace=True) - - self._x_walk = pd.Series(dtype=object) - with open(self.crosswalk) as fp: - data = json.load(fp) - for id, values in data.items(): - gage = values.get('Gage_no') - if gage: - if not isinstance(gage, str): - gage = gage[0] - if gage != "": - self._x_walk[id] = gage + self._read_legacy_hydrofabric() #Read the calibration specific info with open(self.realization) as fp: data = json.load(fp) self.ngen_realization = NgenRealization(**data) + def _read_hydrofabric(self): + # Read geopackage hydrofabric + self._catchment_hydro_fabric = gpd.read_file(self.hydrofabric, layer='divides') + self._catchment_hydro_fabric.set_index('divide_id', inplace=True) + + self._nexus_hydro_fabric = gpd.read_file(self.hydrofabric, layer='nexus') + self._nexus_hydro_fabric.set_index('id', inplace=True) + + self._flowpath_hydro_fabric = gpd.read_file(self.hydrofabric, layer='flowpaths') + self._flowpath_hydro_fabric.set_index('id', inplace=True) + + attributes = gpd.read_file(self.hydrofabric, layer="flowpath_attributes") + + self._x_walk = pd.Series( attributes[ ~ attributes['rl_gages'].isna() ]['rl_gages'] ) + + def _read_legacy_hydrofabric(self): + # Legacy geojson support + assert self.catchments is not None, "missing geojson catchments file" + assert self.nexus is not None, "missing geojson nexus file" + assert self.crosswalk is not None, "missing crosswalk file" + self._catchment_hydro_fabric = gpd.read_file(self.catchments) + self._catchment_hydro_fabric = self._catchment_hydro_fabric.rename(columns=str.lower) + self._catchment_hydro_fabric.set_index('id', inplace=True) + self._nexus_hydro_fabric = gpd.read_file(self.nexus) + self._nexus_hydro_fabric = self._nexus_hydro_fabric.rename(columns=str.lower) + self._nexus_hydro_fabric.set_index('id', inplace=True) + + self._x_walk = pd.Series(dtype=object) + with open(self.crosswalk) as fp: + data = json.load(fp) + for id, values in data.items(): + gage = values.get('Gage_no') + if gage: + if not isinstance(gage, str): + gage = gage[0] + if gage != "": + self._x_walk[id] = gage + @property def config_file(self) -> Path: """Path to the configuration file for this calibration From a69285692e67d1f5ddc1b655edde8d933842e6ac Mon Sep 17 00:00:00 2001 From: Austin Raney Date: Mon, 29 Jul 2024 21:22:24 -0400 Subject: [PATCH 2/3] fix: handle legacy gpkg and >2.1 hfs separately --- python/ngen_cal/src/ngen/cal/ngen.py | 47 +++++++++++++++++++++++++--- 1 file changed, 43 insertions(+), 4 deletions(-) diff --git a/python/ngen_cal/src/ngen/cal/ngen.py b/python/ngen_cal/src/ngen/cal/ngen.py index f8ae167f..ce05a9a0 100644 --- a/python/ngen_cal/src/ngen/cal/ngen.py +++ b/python/ngen_cal/src/ngen/cal/ngen.py @@ -117,16 +117,52 @@ def __init__(self, **kwargs): # Read the catchment hydrofabric data if self.hydrofabric is not None: - self._read_hydrofabric() + if self._is_legacy_gpkg_hydrofabric(self.hydrofabric): + self._read_legacy_gpkg_hydrofabric() + else: + self._read_gpkg_hydrofabric() else: - self._read_legacy_hydrofabric() + self._read_legacy_geojson_hydrofabric() #Read the calibration specific info with open(self.realization) as fp: data = json.load(fp) self.ngen_realization = NgenRealization(**data) - def _read_hydrofabric(self): + @staticmethod + def _is_legacy_gpkg_hydrofabric(hydrofabric: Path) -> bool: + """Return True if legacy hydrofabric.""" + import sqlite3 + connection = sqlite3.connect(hydrofabric) + # hydrofabric <= 2.1 use 'flowpaths' + # hydrofabric > 2.1 use 'flowlines' + query = "SELECT name FROM sqlite_master WHERE type='table' AND name='flowpaths';" + try: + cursor = connection.execute(query) + value = cursor.fetchone() + finally: + connection.close() + return value is not None + + def _read_gpkg_hydrofabric(self) -> None: + # Read geopackage hydrofabric + self._catchment_hydro_fabric = gpd.read_file(self.hydrofabric, layer='divides') + self._catchment_hydro_fabric.set_index('divide_id', inplace=True) + + self._nexus_hydro_fabric = gpd.read_file(self.hydrofabric, layer='nexus') + self._nexus_hydro_fabric.set_index('id', inplace=True) + + # hydrofabric > 2.1 use 'flowlines' + self._flowpath_hydro_fabric = gpd.read_file(self.hydrofabric, layer='flowlines') + self._flowpath_hydro_fabric.set_index('id', inplace=True) + + # hydrofabric > 2.1 use 'flowpath-attributes' + attributes = gpd.read_file(self.hydrofabric, layer="flowpath-attributes") + attributes.set_index("id", inplace=True) + + self._x_walk = pd.Series( attributes[ ~ attributes['rl_gages'].isna() ]['rl_gages'] ) + + def _read_legacy_gpkg_hydrofabric(self) -> None: # Read geopackage hydrofabric self._catchment_hydro_fabric = gpd.read_file(self.hydrofabric, layer='divides') self._catchment_hydro_fabric.set_index('divide_id', inplace=True) @@ -134,14 +170,17 @@ def _read_hydrofabric(self): self._nexus_hydro_fabric = gpd.read_file(self.hydrofabric, layer='nexus') self._nexus_hydro_fabric.set_index('id', inplace=True) + # hydrofabric <= 2.1 use 'flowpaths' self._flowpath_hydro_fabric = gpd.read_file(self.hydrofabric, layer='flowpaths') self._flowpath_hydro_fabric.set_index('id', inplace=True) + # hydrofabric <= 2.1 use 'flowpath_attributes' attributes = gpd.read_file(self.hydrofabric, layer="flowpath_attributes") + attributes.set_index("id", inplace=True) self._x_walk = pd.Series( attributes[ ~ attributes['rl_gages'].isna() ]['rl_gages'] ) - def _read_legacy_hydrofabric(self): + def _read_legacy_geojson_hydrofabric(self) -> None: # Legacy geojson support assert self.catchments is not None, "missing geojson catchments file" assert self.nexus is not None, "missing geojson nexus file" From f88c9193b6dd47faca9afe72b6cc12d3366e1463 Mon Sep 17 00:00:00 2001 From: Austin Raney Date: Thu, 1 Aug 2024 10:14:33 -0400 Subject: [PATCH 3/3] doc(docstring): improve specificity Co-authored-by: Robert Bartel --- python/ngen_cal/src/ngen/cal/ngen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ngen_cal/src/ngen/cal/ngen.py b/python/ngen_cal/src/ngen/cal/ngen.py index ce05a9a0..d4f7a4c4 100644 --- a/python/ngen_cal/src/ngen/cal/ngen.py +++ b/python/ngen_cal/src/ngen/cal/ngen.py @@ -131,7 +131,7 @@ def __init__(self, **kwargs): @staticmethod def _is_legacy_gpkg_hydrofabric(hydrofabric: Path) -> bool: - """Return True if legacy hydrofabric.""" + """Return True if legacy (<=v2.1) gpkg hydrofabric.""" import sqlite3 connection = sqlite3.connect(hydrofabric) # hydrofabric <= 2.1 use 'flowpaths'