Merge branch 'main' into generate-input-based-on-LHM4.3

.github/workflows/docs.yml

@@ -29,7 +29,7 @@ jobs:
 
       - uses: prefix-dev/[email protected]
         with:
-          manifest-path: Ribasim/pixi.toml
+          manifest-path: Ribasim-NL/pixi.toml
           pixi-version: "latest"
 
       - name: Check Quarto installation and all engines

.github/workflows/pre-commit_auto_update.yml

@@ -18,7 +18,7 @@ jobs:
         with:
           python-version: "3.11"
       - uses: browniebroke/pre-commit-autoupdate-action@main
-      - uses: peter-evans/create-pull-request@v6
+      - uses: peter-evans/create-pull-request@v7
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           branch: update/pre-commit-hooks

.gitignore

@@ -20,17 +20,13 @@ dmypy.json
 # pytest --basetemp
 src/hydamo/tests/temp/
 src/ribasim_nl/tests/temp/
-src/peilbeheerst_model/tests/temp/
 
 src/peilbeheerst_model/*.html
 src/peilbeheerst_model/*.code-workspace
 src/peilbeheerst_model/.vscode
 
-src/peilbeheerst_model/*.jpeg
-src/peilbeheerst_model/*.gpkg
-src/peilbeheerst_model/tests_results
-/src/peilbeheerst_model/Output_zdrive
-/src/peilbeheerst_model/Rekenend_Model_Test
-/src/peilbeheerst_model/vervallen
+/src/peilbeheerst_model/Parametrize/ribasim
+/src/peilbeheerst_model/Parametrize/fix_FF_HHSK.ipynb
+/src/peilbeheerst_model/01_test_parse_crossings.ipynb
 
 notebooks/rijkswaterstaat/plots/

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.6.0
+    rev: v5.0.0
     hooks:
       - id: check-added-large-files
       - id: check-case-conflict
@@ -14,7 +14,7 @@ repos:
         exclude: '.teamcity'
       - id: trailing-whitespace
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.5.5
+    rev: v0.7.1
     hooks:
       - id: ruff
         types_or: [python, pyi, jupyter]

docs/_quarto.yml

@@ -9,11 +9,19 @@ website:
     left:
       - reference/index.qmd
       - cloudstorage.qmd
+      - text: "Workflows"
+        file: workflow/main-watersystem.qmd
     right:
       - icon: github
         href: https://github.com/Deltares/Ribasim-NL
         aria-label: GitHub
 
+  sidebar:
+    - title: "Workflows"
+      contents:
+        - workflow/main-watersystem.qmd
+        - workflow/level-controlled.qmd
+
 format:
   html:
     theme: cosmo

docs/workflow/level-controlled.qmd

@@ -0,0 +1,22 @@
+---
+title: "Level controlled"
+---
+
+Workflow for deriving level controlled (peilbeheerst) regional models.
+All code can be found under [`src/peilbeheerst_model`](https://github.com/Deltares/Ribasim-NL/tree/main/src/peilbeheerst_model).
+The paths below are relative to this path.
+
+1. Run the preprocessing scripts. One script per water board, path: `peilbeheerst_model/preprocess_data/`
+2. Run the postprocessing script. One script per water board, path: `peilbeheerst_model/postprocess_data/`
+3. Run the crossings script. One script, path: `01_parse_crossings.ipynb`. Moving to one script per water board under `peilbeheerst_model/parse_crossings/`.
+4. Run shortest paths scripts. One script per water board, path: `Shortest_path/`
+5. Run crossings to Ribasim script. One script, all water boards are below each other, path: `02_crossings_to_ribasim_notebook.ipynb`. Moving to one script per water board under `peilbeheerst_model/crossings_to_ribasim/`.
+6. Run parametrize scripts. One script per water board, for now only Amstel, Gooi en Vecht (AGV), path: `Parametrize/AmstelGooienVecht_parametrize.ipynb`
+
+We originally had more parametrize scripts, but because so much has changed I have now saved these in our backup.
+We will only use these to see if there were any additional manual adjustments.
+For the rest, it follows the same workflow as `AmstelGooienVecht_parametrize.ipynb`.
+
+Finally: step 1 started with a clear script per water board.
+During the process of 1.5 years, these scripts have become increasingly larger and more confusing, whereby not every line is needed anymore.
+For now, there is no priority to clean this up, partly because this is a major risk that the data will (unintentionally) change, which will change the networks and the feedback forms can no longer be used.

docs/workflow/main-watersystem.qmd

@@ -0,0 +1,17 @@
+---
+title: "Main watersystem"
+---
+
+All files are in [`notebooks/rijkswaterstaat`](https://github.com/Deltares/Ribasim-NL/tree/main/notebooks/rijkswaterstaat) are numbered in order of execution.
+
+# 1_bathymetrie.py
+In this script we merge data from Baseline (version j23_6-v1, non-public) with grids from [bathymetrie-Nederland](https://maps.rijkswaterstaat.nl/geoweb55/index.html?viewer=Bathymetrie_Nederland) to one 5x5m GeoTiff.
+
+We combine both sources as the Midden Limburgse en Noord Brabantse (MLNB) kanalen are not in Baseline and the IJsselmeer is not completely covered in bathymetrie-Nederland.
+For the mask we use Top10NL waterdelen and manually filled a boolean column `baseline`.
+
+::: {layout-ncol=2}
+![Bathymetry mask](https://s3.deltares.nl/ribasim/doc-image/ribasim-nl/bathymetrie_mask.png)
+
+![Bathymetry](https://s3.deltares.nl/ribasim/doc-image/ribasim-nl/bathymetrie_result.png)
+:::

notebooks/aa_en_maas/00_get_model.py

@@ -0,0 +1,17 @@
+# %%
+from ribasim_nl import CloudStorage
+
+cloud = CloudStorage()
+
+authority = "AaenMaas"
+short_name = "aam"
+
+cloud = CloudStorage()
+
+model_url = cloud.joinurl(authority, "modellen", f"{authority}_2024_6_3")
+ribasim_toml = cloud.joinpath(authority, "modellen", f"{authority}_2024_6_3", "model.toml")
+if not ribasim_toml.exists():
+    cloud.download_content(model_url)
+
+if ribasim_toml.exists():  # get a short_name version to differentiate QGIS layergroup
+    ribasim_toml.with_name(f"{short_name}.toml").write_text(ribasim_toml.read_text())

notebooks/aa_en_maas/00_review_model_network.py

@@ -0,0 +1,59 @@
+# %%
+
+import pandas as pd
+
+from ribasim_nl import CloudStorage, Model, NetworkValidator
+
+cloud = CloudStorage()
+
+authority = "AaenMaas"
+short_name = "aam"
+
+ribasim_toml = cloud.joinpath(authority, "modellen", f"{authority}_2024_6_3", f"{short_name}.toml")
+database_gpkg = ribasim_toml.with_name("database.gpkg")
+
+model = Model.read(ribasim_toml)
+network_validator = NetworkValidator(model)
+
+verwerkt_dir = cloud.joinpath(authority, "verwerkt")
+verwerkt_dir.mkdir(exist_ok=True)
+
+modelfouten_gpkg = cloud.joinpath(authority, "verwerkt", "modelfouten.gpkg")
+
+# %% verwijderen duplicated edges
+
+duplicated_edges = len(model.edge.df[model.edge.df.duplicated()])
+model.edge.df.drop_duplicates(inplace=True)
+
+# %% wegschrijven fouten
+
+# niet-bestaande fouten
+mask = model.edge.df.to_node_id.isin(model.node_table().df.index) & model.edge.df.from_node_id.isin(
+    model.node_table().df.index
+)
+
+edge_mist_node_df = model.edge.df[~mask]
+model.edge.df = model.edge.df[mask]
+
+mask = model.edge.df.geometry.length == 0
+model.edge.df[mask].centroid.to_file(modelfouten_gpkg, layer="edge_zonder_lengte")
+model.edge.df = model.edge.df[~mask]
+
+# niet-gekoppelde areas
+model.basin.area.df[~model.basin.area.df.node_id.isin(model.basin.node.df.index)].to_file(
+    modelfouten_gpkg, layer="area_niet_een_basin"
+)
+
+model.basin.node.df[~model.basin.node.df.index.isin(model.basin.area.df.node_id)].to_file(
+    modelfouten_gpkg, layer="basin_zonder_area"
+)
+
+# ontbrekende basins
+network_validator.node_invalid_connectivity().to_file(modelfouten_gpkg, layer="node_mist")
+pd.concat([network_validator.edge_incorrect_connectivity(), edge_mist_node_df]).to_file(
+    modelfouten_gpkg, layer="ege_mist_node"
+)
+
+# nodes met verkeerde richting
+
+model.invalid_topology_at_node().to_file(modelfouten_gpkg, layer="node_met_verkeerde_instroom_uitstroom_egde")