Merge pull request #337 from Dewberry/release/0.10.1

Bugfix Release 0.10.1

docs/source/postman.rst

@@ -3,6 +3,10 @@ Postman collection
 
 For reference and documentation of the API, please open the postman collection for the version of ripple1d
 
+`v.0.10.1: <https://github.com/Dewberry/ripple1d/blob/58a873910f0dfe312f7d674793470389836aac5b/ripple1d/api/postman_collection.json>`_ This version contains new args for the conflate_model and compute_conflation_metrics endpoints :
+ - `model_name` (str)  added to  `extract_submodel`.  This is the name of the source model. Example: Red River.prj -> Red River (model_name)
+ - `terrain_agreement_ignore_error` (bool)  added to  `create_ras_terrain`.  If true, this will log and ignore any errors encountered in the terrain agreement calculation process.
+
 `v.0.10.0: <https://github.com/Dewberry/ripple1d/blob/93cf22cf11791d59820635be6c02327b39912b49/ripple1d/api/postman_collection.json>`_ This version contains new args for the conflate_model and compute_conflation_metrics endpoints :
  - `model_name` (str)  added to  `conflate_model`.  This is the name of the source model. Example: Red River.prj -> Red River (model_name)
  - `model_name` (str)  added to  `compute_conflation_metrics`.  This is the name of the source model.Example: Red River.prj -> Red River (model_name)

ripple1d/api/postman_collection.json

@@ -1,10 +1,10 @@
 {
 	"info": {
-		"_postman_id": "977032b0-6c0b-4437-9347-7061b817bdae",
+		"_postman_id": "aae308a3-7a55-4367-a5f4-8728c9165d68",
 		"name": "ripple1d",
 		"description": "Collection for processing existing HEC-RAS models for use in the production of Flood Inundation Maps (FIMs) and rating curves for use in near-real time flood forecasting on the NOAA National Water Model",
 		"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json",
-		"_exporter_id": "29128857"
+		"_exporter_id": "38126273"
 	},
 	"item": [
 		{
@@ -75,7 +75,7 @@
 						"header": [],
 						"body": {
 							"mode": "raw",
-							"raw": "{\r\n    \"source_model_directory\": \"{{source_model_directory}}\",\r\n    \"model_name\":\"{{source_model_name}}\",\r\n    \"source_network\": {\"file_name\":\"{{nwm_data_directory}}\\\\flows.parquet\",\r\n                        \"version\":\"2.1\",  // optional\r\n                        \"type\":\"nwm_hydrofabric\"}\r\n    \r\n}",
+							"raw": "{\r\n    \"source_model_directory\": \"{{source_model_directory}}\",\r\n    \"model_name\":\"{{source_model_name}}\",\r\n    \"source_network\": {\"file_name\":\"{{nwm_data_directory}}\\\\nwm_bbox.parquet\",\r\n                        \"version\":\"2.1\",  // optional\r\n                        \"type\":\"nwm_hydrofabric\"}\r\n    \r\n}",
 							"options": {
 								"raw": {
 									"language": "json"
@@ -104,7 +104,7 @@
 						"header": [],
 						"body": {
 							"mode": "raw",
-							"raw": "{\r\n    \"source_model_directory\": \"{{source_model_directory}}\",\r\n    \"submodel_directory\": \"{{submodels_base_directory}}\\\\{{nwm_reach_id}}\",\r\n    \"nwm_id\": \"{{nwm_reach_id}}\"\r\n}",
+							"raw": "{\r\n    \"source_model_directory\": \"{{source_model_directory}}\",\r\n    \"submodel_directory\": \"{{submodels_base_directory}}\\\\{{nwm_reach_id}}\",\r\n    \"nwm_id\": \"{{nwm_reach_id}}\",\r\n    \"model_name\":\"{{source_model_name}}\"\r\n}",
 							"options": {
 								"raw": {
 									"language": "json"
@@ -133,7 +133,7 @@
 						"header": [],
 						"body": {
 							"mode": "raw",
-							"raw": "{\r\n    \"source_model_directory\": \"{{source_model_directory}}\",\r\n    \"crs\": 2227,\r\n    \"metadata\": {\"stac_api\":\"https://stac2.dewberryanalytics.com\",  // optional\r\n                \"stac_collection_id\":\"ebfe-12090301_LowerColoradoCummins\",  // optional\r\n                \"stac_item_id\":\"137a9667-e5cf-4cea-b6ec-2e882a42fdc8\"}  // optional\r\n}\r\n",
+							"raw": "{\r\n    \"source_model_directory\": \"{{source_model_directory}}\",\r\n    \"crs\": 3452,\r\n    \"metadata\": {\"stac_api\":\"https://stac2.dewberryanalytics.com\",  // optional\r\n                \"stac_collection_id\":\"ebfe-12090301_LowerColoradoCummins\",  // optional\r\n                \"stac_item_id\":\"137a9667-e5cf-4cea-b6ec-2e882a42fdc8\"}  // optional\r\n}\r\n",
 							"options": {
 								"raw": {
 									"language": "json"
@@ -162,7 +162,7 @@
 						"header": [],
 						"body": {
 							"mode": "raw",
-							"raw": "{\r\n    \"submodel_directory\": \"{{submodels_base_directory}}\\\\{{nwm_reach_id}}\",\r\n    \"resolution\": 3,\r\n    \"resolution_units\": \"Meters\",\r\n    \"terrain_agreement_resolution\": 3\r\n}",
+							"raw": "{\r\n    \"submodel_directory\": \"{{submodels_base_directory}}\\\\{{nwm_reach_id}}\",\r\n    \"resolution\": 3,\r\n    \"resolution_units\": \"Meters\",\r\n    \"terrain_agreement_resolution\": 3,\r\n    \"terrain_agreement_ignore_error\": false\r\n}",
 							"options": {
 								"raw": {
 									"language": "json"

ripple1d/conflate/rasfim.py

@@ -232,17 +232,6 @@ def load_pq(self, nwm_pq: str):
         try:
             # read nwm reaches with bbox
             nwm_reaches = gpd.read_parquet(nwm_pq, bbox=self._ras_xs.to_crs(self.common_crs).total_bounds)
-            # select subset of nwm reaches using concave hull of cross sections
-            cch = gpd.GeoDataFrame(
-                {
-                    "geometry": [
-                        self._ras_xs.to_crs(self.common_crs).dissolve("river_reach").convex_hull.buffer(100).union_all()
-                    ]
-                },
-                geometry="geometry",
-                crs=self.common_crs,
-            )
-            nwm_reaches = nwm_reaches.loc[nwm_reaches.intersects(cch.iloc[0].geometry)]
 
             # rename geometry
             nwm_reaches = nwm_reaches.rename(columns={"geom": "geometry"})
@@ -468,7 +457,7 @@ def endpoints_from_multiline(mline: MultiLineString) -> Tuple[Point, Point]:
 
 
 def nearest_line_to_point(
-    lines: gpd.GeoDataFrame, point: Point, column_id: str = "ID", search_radius: int = 1e9, number_of_returns: int = 1
+    lines: gpd.GeoDataFrame, point: Point, column_id: str = "ID", search_radius: int = 1e9, number_of_returns: int = 0
 ) -> np.array:
     """
     Return the ID of the line(s) closest to the point.

ripple1d/data_model.py

@@ -187,15 +187,11 @@ def nwm_conflation_parameters(self, nwm_id: str):
 class RippleSourceDirectory:
     """Source Directory for Ripple to create NwmReachModel's. Should contain the conflation.json file and gpkg file for the source model."""
 
-    def __init__(self, source_directory: str):
+    def __init__(self, source_directory: str, model_name: str):
 
         self.source_directory = source_directory
         self.model_basename = os.path.basename(self.source_directory)
-
-    @property
-    def model_name(self):
-        """Model name."""
-        return self.model_basename
+        self.model_name = model_name
 
     def derive_path(self, extension: str):
         """Derive path."""

ripple1d/ops/ras_conflate.py

@@ -1,18 +1,12 @@
 """Conflate HEC-RAS Model."""
 
-import copy
 import json
 import logging
 import os
 import traceback
-from datetime import datetime
 from itertools import chain, permutations
 from urllib.parse import quote
 
-import boto3
-import pandas as pd
-import pystac
-
 import ripple1d
 from ripple1d.conflate.rasfim import (
     RasFimConflater,
@@ -23,7 +17,7 @@
 )
 from ripple1d.errors import InvalidNetworkPath
 from ripple1d.ops.metrics import compute_conflation_metrics
-from ripple1d.utils.ripple_utils import NWMWalker, clip_ras_centerline
+from ripple1d.utils.ripple_utils import clip_ras_centerline
 
 logging.getLogger("fiona").setLevel(logging.ERROR)
 logging.getLogger("botocore").setLevel(logging.ERROR)
@@ -131,8 +125,8 @@ def conflate_model(source_model_directory: str, model_name: str, source_network:
     threshold listed for the reach in the NWM network.  The high flow is the
     100 year flow from the NWM network.
     """
-    logging.info(f"conflate_model starting")
-    if not "file_name" in source_network:
+    logging.info("conflate_model starting")
+    if "file_name" not in source_network:
         raise KeyError(f"source_network must contain 'file_name', invalid parameters: {source_network}")
 
     if not source_network["type"] == "nwm_hydrofabric":
@@ -152,7 +146,7 @@ def conflate_model(source_model_directory: str, model_name: str, source_network:
             logging.error(f"| Error: {e}")
             logging.error(f"| Traceback: {traceback.format_exc()}")
 
-        logging.info(f"conflate_model complete")
+        logging.info("conflate_model complete")
     return {"conflation_file": conflation_file}
 
 
@@ -168,7 +162,7 @@ def _conflate_model(source_model_directory: str, model_name: str, source_network
         "metadata": generate_metadata(source_network, rfc),
     }
     conflation = find_eclipsed_reaches(rfc, conflation)
-    # conflation = fix_junctions(rfc, conflation)
+    conflation = fix_junctions(rfc, conflation)
     return conflation
 
 
@@ -179,7 +173,7 @@ def find_eclipsed_reaches(rfc: RasFimConflater, conflation: dict) -> dict:
         try:
             reaches = rfc.nwm_walker.walk(us_reach, ds_reach)
             for r in reaches:
-                if not r in [us_reach, ds_reach]:
+                if r not in [us_reach, ds_reach]:
                     conflation["reaches"][r] = {"eclipsed": True} | rfc.get_nwm_reach_metadata(r)
         except InvalidNetworkPath as e:
             logging.error(f"|Error: {e}")

ripple1d/ops/ras_terrain.py

@@ -6,6 +6,7 @@
 import logging
 import os
 import re
+import traceback
 from math import ceil, comb, pi
 from pathlib import Path
 
@@ -66,6 +67,7 @@ def create_ras_terrain(
     terrain_agreement_el_repeats: int = 5,
     terrain_agreement_el_ramp_rate: float = 2.0,
     terrain_agreement_el_init: float = 0.5,
+    terrain_agreement_ignore_error: bool = True,
 ):
     """Create a RAS terrain file.
 
@@ -105,6 +107,8 @@ def create_ras_terrain(
         21.5, etc.
     terrain_agreement_el_init : float, optional
         initial value for terrain agreement elevation increments, by default 0.5
+    terrain_agreement_ignore_error : bool, optional
+        whether to raise or log+ignore any errors encountered in the compute_terrain_agreement_metrics function.
     task_id : str, optional
         Task ID to use for logging, by default ""
 
@@ -195,17 +199,25 @@ def create_ras_terrain(
     logging.info(f"create_ras_terrain complete")
 
     # Calculate terrain agreement metrics
-    agreement_path = compute_terrain_agreement_metrics(
-        submodel_directory,
-        terrain_path,
-        terrain_agreement_resolution,
-        resolution_units,
-        terrain_agreement_format,
-        terrain_agreement_el_repeats,
-        terrain_agreement_el_ramp_rate,
-        terrain_agreement_el_init,
-    )
-    result["terrain_agreement"] = agreement_path
+    try:
+        agreement_path = compute_terrain_agreement_metrics(
+            submodel_directory,
+            terrain_path,
+            terrain_agreement_resolution,
+            resolution_units,
+            terrain_agreement_format,
+            terrain_agreement_el_repeats,
+            terrain_agreement_el_ramp_rate,
+            terrain_agreement_el_init,
+        )
+        result["terrain_agreement"] = agreement_path
+    except Exception as e:
+        if not terrain_agreement_ignore_error:
+            raise e
+        logging.error(f"| Error: {e}")
+        logging.error(f"| Traceback: {traceback.format_exc()}")
+
+        result["terrain_agreement"] = None
     return result
 
 

ripple1d/ops/subset_gpkg.py

@@ -46,7 +46,7 @@ def source_model_metadata(self):
         """Metadata from the source model."""
         with sqlite3.connect(self.src_gpkg_path) as conn:
             cur = conn.cursor()
-            res = cur.execute(f"SELECT key,value from metadata")
+            res = cur.execute("SELECT key,value from metadata")
             return dict(res.fetchall())
 
     def copy_metadata_to_ripple1d_gpkg(self):
@@ -96,7 +96,10 @@ def us_rs(self) -> str:
     @property
     def us_river_reach(self) -> str:
         """Extract upstream river_reach from conflation parameters."""
-        return f"{self.us_river.ljust(16)},{self.us_reach.ljust(16)}"
+        row = self.source_river[
+            (self.source_river["river"] == self.us_river) & (self.source_river["reach"] == self.us_reach)
+        ]
+        return row.iloc[0]["river_reach"]
 
     @property
     def us_river_reach_rs(self) -> str:
@@ -121,7 +124,10 @@ def ds_rs(self) -> str:
     @property
     def ds_river_reach(self) -> str:
         """Extract downstream river_reach from conflation parameters."""
-        return f"{self.ds_river.ljust(16)},{self.ds_reach.ljust(16)}"
+        row = self.source_river[
+            (self.source_river["river"] == self.ds_river) & (self.source_river["reach"] == self.ds_reach)
+        ]
+        return row.iloc[0]["river_reach"]
 
     @property
     def us_river_reach_rs(self) -> str:
@@ -206,7 +212,7 @@ def ripple_xs_concave_hull(self):
         try:
             hulls = self.split_source_hull
             ripple_xs_concave_hull = gpd.GeoDataFrame({"geometry": hulls}, geometry="geometry", crs=self.crs)
-        except Exception as e:
+        except Exception:
             ripple_xs_concave_hull = xs_concave_hull(fix_reversed_xs(self.ripple_xs, self.ripple_river))
         return ripple_xs_concave_hull
 
@@ -441,7 +447,7 @@ def write_ripple_gpkg(self) -> None:
             if layer == "Structure":
                 if (gdf["type"] == 6).any():
                     logging.warning(
-                        f"Lateral structures are not currently supported in ripple1d. The lateral structures will be dropped."
+                        "Lateral structures are not currently supported in ripple1d. The lateral structures will be dropped."
                     )
                     gdf = gdf.loc[gdf["type"] != 6, :]
 
@@ -495,7 +501,7 @@ def write_ripple1d_parameters(self, ripple1d_parameters: dict):
             json.dump(ripple1d_parameters, f, indent=4)
 
 
-def extract_submodel(source_model_directory: str, submodel_directory: str, nwm_id: int):
+def extract_submodel(source_model_directory: str, submodel_directory: str, nwm_id: int, model_name: str):
     """Use ripple conflation data to create a new GPKG from an existing ras geopackage.
 
     Create a new geopackage with information for a specific NWM reach.  The new geopackage contains layer for the river centerline, cross-sections, and structures.
@@ -508,6 +514,8 @@ def extract_submodel(source_model_directory: str, submodel_directory: str, nwm_i
         The path to export submodel HEC-RAS files to.
     nwm_id : int
         The id of the NWM reach to create a submodel for
+    model_name : str
+        The name of the HEC-RAS model.
     task_id : str, optional
         Task ID to use for logging, by default ""
 
@@ -527,7 +535,7 @@ def extract_submodel(source_model_directory: str, submodel_directory: str, nwm_i
         raise FileNotFoundError(
             f"cannot find directory for source model {source_model_directory}, please ensure dir exists"
         )
-    rsd = RippleSourceDirectory(source_model_directory)
+    rsd = RippleSourceDirectory(source_model_directory, model_name)
 
     logging.info(f"extract_submodel starting for nwm_id {nwm_id}")
 

ripple1d/version.py

@@ -1,3 +1,3 @@
 """ripple1d version."""
 
-__version__ = "0.10.0"
+__version__ = "0.10.1"

tests/api_tests.py

@@ -102,6 +102,7 @@ def test_b_extract_submodel(self):
             "source_model_directory": self.SOURCE_RAS_MODEL_DIRECTORY,
             "submodel_directory": self.SUBMODELS_DIRECTORY,
             "nwm_id": self.REACH_ID,
+            "model_name": self.MODEL_NAME,
         }
         process = "extract_submodel"
         files = [self.GPKG_FILE]

tests/conflation_tests/plotting.py

@@ -128,14 +128,19 @@ def divide_section(self, line, splits):
         return subs
 
     def clip_xs(self, subset, overlaps, r):
-        for xs_id in subset["river_reach_rs"].values:
+        for xs_id in subset["source_river_reach_rs"]:
             if xs_id in list(overlaps.keys()):
                 splits = len(overlaps[xs_id])
                 position = overlaps[xs_id].index(r)
-                geom = subset.loc[subset["river_reach_rs"] == xs_id, "geometry"].values[0]
-                subset.loc[subset["river_reach_rs"] == xs_id, "geometry"] = self.divide_section(geom, splits)[position]
+                geom = subset.loc[subset["source_river_reach_rs"] == xs_id, "geometry"].values[0]
+                subset.loc[subset["source_river_reach_rs"] == xs_id, "geometry"] = self.divide_section(geom, splits)[
+                    position
+                ]
         return subset
 
+    def format_source_id(self, row: dict) -> str:
+        return " ".join([str(row["source_river"]), str(row["source_reach"]), str(row["source_river_station"])])
+
     def generate_subset_gdfs(self) -> dict:
         # Create combo gdf
         subset_gdfs = {}
@@ -145,10 +150,11 @@ def generate_subset_gdfs(self) -> dict:
                 continue
             subsetter = RippleGeopackageSubsetter(self.ras_path, self.conflation_path, None, r)
             subset = subsetter.subset_xs
+            subset["source_river_reach_rs"] = subset.apply(self.format_source_id, axis=1)
             subset_gdfs[r] = subset
 
             # log xs for duplicate detection
-            for xs_id in subset["river_reach_rs"]:
+            for xs_id in subset["source_river_reach_rs"]:
                 overlaps[xs_id].append(r)
 
         # split duplicated section geometry
@@ -207,7 +213,7 @@ def apply_formatting(self):
             ax.legend(fontsize="x-small", loc="lower right", ncols=len(self.nwm_reaches) + 1)
             ax.set_axis_off()
             try:
-                ctx.add_basemap(ax, crs=self.crs, source=ctx.providers.USGS.USTopo, attribution=False)
+                ctx.add_basemap(ax, crs=self.crs, source=ctx.providers.USGS.USImagery, attribution=False)
             except requests.exceptions.HTTPError as e:
                 try:
                     ctx.add_basemap(ax, crs=self.crs, source=ctx.providers.Esri.WorldStreetMap, attribution=False)

tests/conftest.py

@@ -26,6 +26,7 @@ def setup_data(request):
     SOURCE_RAS_MODEL_DIRECTORY = os.path.join(TEST_DIR, f"ras-data\\{RAS_MODEL}")
     SUBMODELS_BASE_DIRECTORY = os.path.join(SOURCE_RAS_MODEL_DIRECTORY, "submodels")
     SUBMODELS_DIRECTORY = os.path.join(SUBMODELS_BASE_DIRECTORY, REACH_ID)
+    SUBMODEL_NAME = REACH_ID
     FIM_LIB_DIRECTORY = os.path.join(SUBMODELS_DIRECTORY, f"fims")
     request.cls.FIM_LIB_DIRECTORY = FIM_LIB_DIRECTORY
     request.cls.REACH_ID = RAS_MODEL