From 3859278d63bf6afbe7075d7d20dc5f1a7583e051 Mon Sep 17 00:00:00 2001 From: sclaw Date: Mon, 3 Mar 2025 09:47:11 -0500 Subject: [PATCH 1/8] gh-327 --- ripple1d/ops/ras_terrain.py | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/ripple1d/ops/ras_terrain.py b/ripple1d/ops/ras_terrain.py index af12fa6..a1cfffa 100644 --- a/ripple1d/ops/ras_terrain.py +++ b/ripple1d/ops/ras_terrain.py @@ -66,6 +66,7 @@ def create_ras_terrain( terrain_agreement_el_repeats: int = 5, terrain_agreement_el_ramp_rate: float = 2.0, terrain_agreement_el_init: float = 0.5, + ignore_error: bool = True, ): """Create a RAS terrain file. @@ -195,17 +196,22 @@ def create_ras_terrain( logging.info(f"create_ras_terrain complete") # Calculate terrain agreement metrics - agreement_path = compute_terrain_agreement_metrics( - submodel_directory, - terrain_path, - terrain_agreement_resolution, - resolution_units, - terrain_agreement_format, - terrain_agreement_el_repeats, - terrain_agreement_el_ramp_rate, - terrain_agreement_el_init, - ) - result["terrain_agreement"] = agreement_path + try: + agreement_path = compute_terrain_agreement_metrics( + submodel_directory, + terrain_path, + terrain_agreement_resolution, + resolution_units, + terrain_agreement_format, + terrain_agreement_el_repeats, + terrain_agreement_el_ramp_rate, + terrain_agreement_el_init, + ) + result["terrain_agreement"] = agreement_path + except Exception as e: + if not ignore_error: + raise e + result["terrain_agreement"] = None return result From ce073d578efe32b49edf6a67d508d5f5c4d68921 Mon Sep 17 00:00:00 2001 From: sclaw Date: Mon, 3 Mar 2025 10:59:17 -0500 Subject: [PATCH 2/8] gh-327 --- ripple1d/ops/ras_terrain.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/ripple1d/ops/ras_terrain.py b/ripple1d/ops/ras_terrain.py index a1cfffa..06ef8a3 100644 --- a/ripple1d/ops/ras_terrain.py +++ b/ripple1d/ops/ras_terrain.py @@ -6,6 +6,7 @@ import logging import os import re +import traceback from math import ceil, comb, pi from pathlib import Path @@ -66,7 +67,7 @@ def create_ras_terrain( terrain_agreement_el_repeats: int = 5, terrain_agreement_el_ramp_rate: float = 2.0, terrain_agreement_el_init: float = 0.5, - ignore_error: bool = True, + terrain_agreement_ignore_error: bool = True, ): """Create a RAS terrain file. @@ -106,6 +107,8 @@ def create_ras_terrain( 21.5, etc. terrain_agreement_el_init : float, optional initial value for terrain agreement elevation increments, by default 0.5 + terrain_agreement_ignore_error : bool, optional + whether to raise or log+ignore any errors encountered in the compute_terrain_agreement_metrics function. task_id : str, optional Task ID to use for logging, by default "" @@ -209,8 +212,11 @@ def create_ras_terrain( ) result["terrain_agreement"] = agreement_path except Exception as e: - if not ignore_error: + if not terrain_agreement_ignore_error: raise e + logging.error(f"| Error: {e}") + logging.error(f"| Traceback: {traceback.format_exc()}") + result["terrain_agreement"] = None return result From 0c61154954aa2f969e191dd46b31c54fd866ab07 Mon Sep 17 00:00:00 2001 From: sclaw Date: Mon, 3 Mar 2025 11:00:36 -0500 Subject: [PATCH 3/8] update postman --- ripple1d/api/postman_collection.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ripple1d/api/postman_collection.json b/ripple1d/api/postman_collection.json index f07a98d..dbc69f3 100644 --- a/ripple1d/api/postman_collection.json +++ b/ripple1d/api/postman_collection.json @@ -1,10 +1,10 @@ { "info": { - "_postman_id": "977032b0-6c0b-4437-9347-7061b817bdae", + "_postman_id": "aae308a3-7a55-4367-a5f4-8728c9165d68", "name": "ripple1d", "description": "Collection for processing existing HEC-RAS models for use in the production of Flood Inundation Maps (FIMs) and rating curves for use in near-real time flood forecasting on the NOAA National Water Model", "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json", - "_exporter_id": "29128857" + "_exporter_id": "38126273" }, "item": [ { @@ -162,7 +162,7 @@ "header": [], "body": { "mode": "raw", - "raw": "{\r\n \"submodel_directory\": \"{{submodels_base_directory}}\\\\{{nwm_reach_id}}\",\r\n \"resolution\": 3,\r\n \"resolution_units\": \"Meters\",\r\n \"terrain_agreement_resolution\": 3\r\n}", + "raw": "{\r\n \"submodel_directory\": \"{{submodels_base_directory}}\\\\{{nwm_reach_id}}\",\r\n \"resolution\": 3,\r\n \"resolution_units\": \"Meters\",\r\n \"terrain_agreement_resolution\": 3,\r\n \"terrain_agreement_ignore_error\": false\r\n}", "options": { "raw": { "language": "json" From 3cb6d13bdcb69f06b4c4ab3f9bcf2566469a2a05 Mon Sep 17 00:00:00 2001 From: Matt Deshotel Date: Mon, 3 Mar 2025 11:25:58 -0600 Subject: [PATCH 4/8] add model_name to extract_submodel endpoint closes #325 --- ripple1d/api/postman_collection.json | 6 +++--- ripple1d/data_model.py | 8 ++------ ripple1d/ops/subset_gpkg.py | 6 ++++-- tests/api_tests.py | 1 + tests/conftest.py | 1 + 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/ripple1d/api/postman_collection.json b/ripple1d/api/postman_collection.json index f07a98d..247cb02 100644 --- a/ripple1d/api/postman_collection.json +++ b/ripple1d/api/postman_collection.json @@ -75,7 +75,7 @@ "header": [], "body": { "mode": "raw", - "raw": "{\r\n \"source_model_directory\": \"{{source_model_directory}}\",\r\n \"model_name\":\"{{source_model_name}}\",\r\n \"source_network\": {\"file_name\":\"{{nwm_data_directory}}\\\\flows.parquet\",\r\n \"version\":\"2.1\", // optional\r\n \"type\":\"nwm_hydrofabric\"}\r\n \r\n}", + "raw": "{\r\n \"source_model_directory\": \"{{source_model_directory}}\",\r\n \"model_name\":\"{{source_model_name}}\",\r\n \"source_network\": {\"file_name\":\"{{nwm_data_directory}}\\\\nwm_bbox.parquet\",\r\n \"version\":\"2.1\", // optional\r\n \"type\":\"nwm_hydrofabric\"}\r\n \r\n}", "options": { "raw": { "language": "json" @@ -104,7 +104,7 @@ "header": [], "body": { "mode": "raw", - "raw": "{\r\n \"source_model_directory\": \"{{source_model_directory}}\",\r\n \"submodel_directory\": \"{{submodels_base_directory}}\\\\{{nwm_reach_id}}\",\r\n \"nwm_id\": \"{{nwm_reach_id}}\"\r\n}", + "raw": "{\r\n \"source_model_directory\": \"{{source_model_directory}}\",\r\n \"submodel_directory\": \"{{submodels_base_directory}}\\\\{{nwm_reach_id}}\",\r\n \"nwm_id\": \"{{nwm_reach_id}}\",\r\n \"model_name\":\"{{source_model_name}}\"\r\n}", "options": { "raw": { "language": "json" @@ -133,7 +133,7 @@ "header": [], "body": { "mode": "raw", - "raw": "{\r\n \"source_model_directory\": \"{{source_model_directory}}\",\r\n \"crs\": 2227,\r\n \"metadata\": {\"stac_api\":\"https://stac2.dewberryanalytics.com\", // optional\r\n \"stac_collection_id\":\"ebfe-12090301_LowerColoradoCummins\", // optional\r\n \"stac_item_id\":\"137a9667-e5cf-4cea-b6ec-2e882a42fdc8\"} // optional\r\n}\r\n", + "raw": "{\r\n \"source_model_directory\": \"{{source_model_directory}}\",\r\n \"crs\": 3452,\r\n \"metadata\": {\"stac_api\":\"https://stac2.dewberryanalytics.com\", // optional\r\n \"stac_collection_id\":\"ebfe-12090301_LowerColoradoCummins\", // optional\r\n \"stac_item_id\":\"137a9667-e5cf-4cea-b6ec-2e882a42fdc8\"} // optional\r\n}\r\n", "options": { "raw": { "language": "json" diff --git a/ripple1d/data_model.py b/ripple1d/data_model.py index e24aa6a..99dbe52 100644 --- a/ripple1d/data_model.py +++ b/ripple1d/data_model.py @@ -187,15 +187,11 @@ def nwm_conflation_parameters(self, nwm_id: str): class RippleSourceDirectory: """Source Directory for Ripple to create NwmReachModel's. Should contain the conflation.json file and gpkg file for the source model.""" - def __init__(self, source_directory: str): + def __init__(self, source_directory: str, model_name: str): self.source_directory = source_directory self.model_basename = os.path.basename(self.source_directory) - - @property - def model_name(self): - """Model name.""" - return self.model_basename + self.model_name = model_name def derive_path(self, extension: str): """Derive path.""" diff --git a/ripple1d/ops/subset_gpkg.py b/ripple1d/ops/subset_gpkg.py index d5f982c..1c771e1 100644 --- a/ripple1d/ops/subset_gpkg.py +++ b/ripple1d/ops/subset_gpkg.py @@ -495,7 +495,7 @@ def write_ripple1d_parameters(self, ripple1d_parameters: dict): json.dump(ripple1d_parameters, f, indent=4) -def extract_submodel(source_model_directory: str, submodel_directory: str, nwm_id: int): +def extract_submodel(source_model_directory: str, submodel_directory: str, nwm_id: int, model_name: str): """Use ripple conflation data to create a new GPKG from an existing ras geopackage. Create a new geopackage with information for a specific NWM reach. The new geopackage contains layer for the river centerline, cross-sections, and structures. @@ -508,6 +508,8 @@ def extract_submodel(source_model_directory: str, submodel_directory: str, nwm_i The path to export submodel HEC-RAS files to. nwm_id : int The id of the NWM reach to create a submodel for + model_name : str + The name of the HEC-RAS model. task_id : str, optional Task ID to use for logging, by default "" @@ -527,7 +529,7 @@ def extract_submodel(source_model_directory: str, submodel_directory: str, nwm_i raise FileNotFoundError( f"cannot find directory for source model {source_model_directory}, please ensure dir exists" ) - rsd = RippleSourceDirectory(source_model_directory) + rsd = RippleSourceDirectory(source_model_directory, model_name) logging.info(f"extract_submodel starting for nwm_id {nwm_id}") diff --git a/tests/api_tests.py b/tests/api_tests.py index 1a2001b..d036ecb 100644 --- a/tests/api_tests.py +++ b/tests/api_tests.py @@ -102,6 +102,7 @@ def test_b_extract_submodel(self): "source_model_directory": self.SOURCE_RAS_MODEL_DIRECTORY, "submodel_directory": self.SUBMODELS_DIRECTORY, "nwm_id": self.REACH_ID, + "model_name": self.MODEL_NAME, } process = "extract_submodel" files = [self.GPKG_FILE] diff --git a/tests/conftest.py b/tests/conftest.py index 56dfdbd..61ba1c0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -26,6 +26,7 @@ def setup_data(request): SOURCE_RAS_MODEL_DIRECTORY = os.path.join(TEST_DIR, f"ras-data\\{RAS_MODEL}") SUBMODELS_BASE_DIRECTORY = os.path.join(SOURCE_RAS_MODEL_DIRECTORY, "submodels") SUBMODELS_DIRECTORY = os.path.join(SUBMODELS_BASE_DIRECTORY, REACH_ID) + SUBMODEL_NAME = REACH_ID FIM_LIB_DIRECTORY = os.path.join(SUBMODELS_DIRECTORY, f"fims") request.cls.FIM_LIB_DIRECTORY = FIM_LIB_DIRECTORY request.cls.REACH_ID = RAS_MODEL From 741cb46ba68ef7301c59daaf9c8e76e45d36c49a Mon Sep 17 00:00:00 2001 From: sclaw Date: Mon, 3 Mar 2025 12:30:55 -0500 Subject: [PATCH 5/8] gh-326 --- ripple1d/ops/subset_gpkg.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/ripple1d/ops/subset_gpkg.py b/ripple1d/ops/subset_gpkg.py index d5f982c..964e99e 100644 --- a/ripple1d/ops/subset_gpkg.py +++ b/ripple1d/ops/subset_gpkg.py @@ -46,7 +46,7 @@ def source_model_metadata(self): """Metadata from the source model.""" with sqlite3.connect(self.src_gpkg_path) as conn: cur = conn.cursor() - res = cur.execute(f"SELECT key,value from metadata") + res = cur.execute("SELECT key,value from metadata") return dict(res.fetchall()) def copy_metadata_to_ripple1d_gpkg(self): @@ -96,7 +96,10 @@ def us_rs(self) -> str: @property def us_river_reach(self) -> str: """Extract upstream river_reach from conflation parameters.""" - return f"{self.us_river.ljust(16)},{self.us_reach.ljust(16)}" + row = self.source_river[ + (self.source_river["river"] == self.us_river) & (self.source_river["reach"] == self.us_reach) + ] + return row.iloc[0]["river_reach"] @property def us_river_reach_rs(self) -> str: @@ -121,7 +124,10 @@ def ds_rs(self) -> str: @property def ds_river_reach(self) -> str: """Extract downstream river_reach from conflation parameters.""" - return f"{self.ds_river.ljust(16)},{self.ds_reach.ljust(16)}" + row = self.source_river[ + (self.source_river["river"] == self.ds_river) & (self.source_river["reach"] == self.ds_reach) + ] + return row.iloc[0]["river_reach"] @property def us_river_reach_rs(self) -> str: @@ -206,7 +212,7 @@ def ripple_xs_concave_hull(self): try: hulls = self.split_source_hull ripple_xs_concave_hull = gpd.GeoDataFrame({"geometry": hulls}, geometry="geometry", crs=self.crs) - except Exception as e: + except Exception: ripple_xs_concave_hull = xs_concave_hull(fix_reversed_xs(self.ripple_xs, self.ripple_river)) return ripple_xs_concave_hull @@ -441,7 +447,7 @@ def write_ripple_gpkg(self) -> None: if layer == "Structure": if (gdf["type"] == 6).any(): logging.warning( - f"Lateral structures are not currently supported in ripple1d. The lateral structures will be dropped." + "Lateral structures are not currently supported in ripple1d. The lateral structures will be dropped." ) gdf = gdf.loc[gdf["type"] != 6, :] From 71106d0b45cb4be2046f4f3435d52f4288101858 Mon Sep 17 00:00:00 2001 From: sclaw Date: Mon, 3 Mar 2025 16:09:11 -0500 Subject: [PATCH 6/8] debug model conflation issues (eclipsed reaches were beign skipped) --- ripple1d/conflate/rasfim.py | 13 +------------ ripple1d/ops/ras_conflate.py | 18 ++++++------------ 2 files changed, 7 insertions(+), 24 deletions(-) diff --git a/ripple1d/conflate/rasfim.py b/ripple1d/conflate/rasfim.py index 31df683..d9daee1 100644 --- a/ripple1d/conflate/rasfim.py +++ b/ripple1d/conflate/rasfim.py @@ -232,17 +232,6 @@ def load_pq(self, nwm_pq: str): try: # read nwm reaches with bbox nwm_reaches = gpd.read_parquet(nwm_pq, bbox=self._ras_xs.to_crs(self.common_crs).total_bounds) - # select subset of nwm reaches using concave hull of cross sections - cch = gpd.GeoDataFrame( - { - "geometry": [ - self._ras_xs.to_crs(self.common_crs).dissolve("river_reach").convex_hull.buffer(100).union_all() - ] - }, - geometry="geometry", - crs=self.common_crs, - ) - nwm_reaches = nwm_reaches.loc[nwm_reaches.intersects(cch.iloc[0].geometry)] # rename geometry nwm_reaches = nwm_reaches.rename(columns={"geom": "geometry"}) @@ -468,7 +457,7 @@ def endpoints_from_multiline(mline: MultiLineString) -> Tuple[Point, Point]: def nearest_line_to_point( - lines: gpd.GeoDataFrame, point: Point, column_id: str = "ID", search_radius: int = 1e9, number_of_returns: int = 1 + lines: gpd.GeoDataFrame, point: Point, column_id: str = "ID", search_radius: int = 1e9, number_of_returns: int = 0 ) -> np.array: """ Return the ID of the line(s) closest to the point. diff --git a/ripple1d/ops/ras_conflate.py b/ripple1d/ops/ras_conflate.py index 0035f7b..f75a79a 100644 --- a/ripple1d/ops/ras_conflate.py +++ b/ripple1d/ops/ras_conflate.py @@ -1,18 +1,12 @@ """Conflate HEC-RAS Model.""" -import copy import json import logging import os import traceback -from datetime import datetime from itertools import chain, permutations from urllib.parse import quote -import boto3 -import pandas as pd -import pystac - import ripple1d from ripple1d.conflate.rasfim import ( RasFimConflater, @@ -23,7 +17,7 @@ ) from ripple1d.errors import InvalidNetworkPath from ripple1d.ops.metrics import compute_conflation_metrics -from ripple1d.utils.ripple_utils import NWMWalker, clip_ras_centerline +from ripple1d.utils.ripple_utils import clip_ras_centerline logging.getLogger("fiona").setLevel(logging.ERROR) logging.getLogger("botocore").setLevel(logging.ERROR) @@ -131,8 +125,8 @@ def conflate_model(source_model_directory: str, model_name: str, source_network: threshold listed for the reach in the NWM network. The high flow is the 100 year flow from the NWM network. """ - logging.info(f"conflate_model starting") - if not "file_name" in source_network: + logging.info("conflate_model starting") + if "file_name" not in source_network: raise KeyError(f"source_network must contain 'file_name', invalid parameters: {source_network}") if not source_network["type"] == "nwm_hydrofabric": @@ -152,7 +146,7 @@ def conflate_model(source_model_directory: str, model_name: str, source_network: logging.error(f"| Error: {e}") logging.error(f"| Traceback: {traceback.format_exc()}") - logging.info(f"conflate_model complete") + logging.info("conflate_model complete") return {"conflation_file": conflation_file} @@ -168,7 +162,7 @@ def _conflate_model(source_model_directory: str, model_name: str, source_network "metadata": generate_metadata(source_network, rfc), } conflation = find_eclipsed_reaches(rfc, conflation) - # conflation = fix_junctions(rfc, conflation) + conflation = fix_junctions(rfc, conflation) return conflation @@ -179,7 +173,7 @@ def find_eclipsed_reaches(rfc: RasFimConflater, conflation: dict) -> dict: try: reaches = rfc.nwm_walker.walk(us_reach, ds_reach) for r in reaches: - if not r in [us_reach, ds_reach]: + if r not in [us_reach, ds_reach]: conflation["reaches"][r] = {"eclipsed": True} | rfc.get_nwm_reach_metadata(r) except InvalidNetworkPath as e: logging.error(f"|Error: {e}") From f5927b2bda5d0720b4fc449e4ded6fbbe65e312e Mon Sep 17 00:00:00 2001 From: sclaw Date: Mon, 3 Mar 2025 16:09:52 -0500 Subject: [PATCH 7/8] update plotting following subset_xs method updates in last version --- tests/conflation_tests/plotting.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/tests/conflation_tests/plotting.py b/tests/conflation_tests/plotting.py index 634c6aa..6f08d21 100644 --- a/tests/conflation_tests/plotting.py +++ b/tests/conflation_tests/plotting.py @@ -128,14 +128,19 @@ def divide_section(self, line, splits): return subs def clip_xs(self, subset, overlaps, r): - for xs_id in subset["river_reach_rs"].values: + for xs_id in subset["source_river_reach_rs"]: if xs_id in list(overlaps.keys()): splits = len(overlaps[xs_id]) position = overlaps[xs_id].index(r) - geom = subset.loc[subset["river_reach_rs"] == xs_id, "geometry"].values[0] - subset.loc[subset["river_reach_rs"] == xs_id, "geometry"] = self.divide_section(geom, splits)[position] + geom = subset.loc[subset["source_river_reach_rs"] == xs_id, "geometry"].values[0] + subset.loc[subset["source_river_reach_rs"] == xs_id, "geometry"] = self.divide_section(geom, splits)[ + position + ] return subset + def format_source_id(self, row: dict) -> str: + return " ".join([str(row["source_river"]), str(row["source_reach"]), str(row["source_river_station"])]) + def generate_subset_gdfs(self) -> dict: # Create combo gdf subset_gdfs = {} @@ -145,10 +150,11 @@ def generate_subset_gdfs(self) -> dict: continue subsetter = RippleGeopackageSubsetter(self.ras_path, self.conflation_path, None, r) subset = subsetter.subset_xs + subset["source_river_reach_rs"] = subset.apply(self.format_source_id, axis=1) subset_gdfs[r] = subset # log xs for duplicate detection - for xs_id in subset["river_reach_rs"]: + for xs_id in subset["source_river_reach_rs"]: overlaps[xs_id].append(r) # split duplicated section geometry @@ -207,7 +213,7 @@ def apply_formatting(self): ax.legend(fontsize="x-small", loc="lower right", ncols=len(self.nwm_reaches) + 1) ax.set_axis_off() try: - ctx.add_basemap(ax, crs=self.crs, source=ctx.providers.USGS.USTopo, attribution=False) + ctx.add_basemap(ax, crs=self.crs, source=ctx.providers.USGS.USImagery, attribution=False) except requests.exceptions.HTTPError as e: try: ctx.add_basemap(ax, crs=self.crs, source=ctx.providers.Esri.WorldStreetMap, attribution=False) From 97d67979ddcbe1445593b4b52b4fc2a0dd43133e Mon Sep 17 00:00:00 2001 From: sclaw Date: Tue, 4 Mar 2025 16:04:06 -0500 Subject: [PATCH 8/8] update for release --- docs/source/postman.rst | 4 ++++ ripple1d/version.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/source/postman.rst b/docs/source/postman.rst index 4d48b25..a39c31c 100644 --- a/docs/source/postman.rst +++ b/docs/source/postman.rst @@ -3,6 +3,10 @@ Postman collection For reference and documentation of the API, please open the postman collection for the version of ripple1d +`v.0.10.1: `_ This version contains new args for the conflate_model and compute_conflation_metrics endpoints : + - `model_name` (str) added to `extract_submodel`. This is the name of the source model. Example: Red River.prj -> Red River (model_name) + - `terrain_agreement_ignore_error` (bool) added to `create_ras_terrain`. If true, this will log and ignore any errors encountered in the terrain agreement calculation process. + `v.0.10.0: `_ This version contains new args for the conflate_model and compute_conflation_metrics endpoints : - `model_name` (str) added to `conflate_model`. This is the name of the source model. Example: Red River.prj -> Red River (model_name) - `model_name` (str) added to `compute_conflation_metrics`. This is the name of the source model.Example: Red River.prj -> Red River (model_name) diff --git a/ripple1d/version.py b/ripple1d/version.py index b7a93fa..1e4c7d8 100644 --- a/ripple1d/version.py +++ b/ripple1d/version.py @@ -1,3 +1,3 @@ """ripple1d version.""" -__version__ = "0.10.0" +__version__ = "0.10.1"