From 1c9a32065c9ade990e940a8f9e7a4ec899828bf3 Mon Sep 17 00:00:00 2001 From: vbrunn Date: Thu, 17 Oct 2024 16:30:07 +0200 Subject: [PATCH 01/11] split mltrain into traindata (enabling extration of preliminary tree map in rast and vect format) and mltrain (doing the random forest training) --- .../m.analyse.trees/m.analyse.trees.html | 3 +- .../r.trees.mltrain/r.trees.mltrain.html | 14 +- .../r.trees.mltrain/r.trees.mltrain.py | 344 ++---------- .../r.trees.traindata/Makefile | 7 + .../r.trees.traindata/r.trees.traindata.html | 43 ++ .../r.trees.traindata/r.trees.traindata.py | 507 ++++++++++++++++++ 6 files changed, 608 insertions(+), 310 deletions(-) create mode 100644 grass-gis-addons/m.analyse.trees/r.trees.traindata/Makefile create mode 100644 grass-gis-addons/m.analyse.trees/r.trees.traindata/r.trees.traindata.html create mode 100644 grass-gis-addons/m.analyse.trees/r.trees.traindata/r.trees.traindata.py diff --git a/grass-gis-addons/m.analyse.trees/m.analyse.trees.html b/grass-gis-addons/m.analyse.trees/m.analyse.trees.html index eee208c..1bb9197 100644 --- a/grass-gis-addons/m.analyse.trees/m.analyse.trees.html +++ b/grass-gis-addons/m.analyse.trees/m.analyse.trees.html @@ -49,5 +49,6 @@

AUTHORS

Momen Mawad, mundialis, mawad at mundialis.de

Lina Krisztian, mundialis, krisztian at mundialis.de - +

Guido Riembauer, mundialis, riembauer at mundialis.de

+

Victoria-Leandra Brunn, mundialis, brunn at mundialis.de

diff --git a/grass-gis-addons/m.analyse.trees/r.trees.mltrain/r.trees.mltrain.html b/grass-gis-addons/m.analyse.trees/r.trees.mltrain/r.trees.mltrain.html index 1af6626..df331a1 100644 --- a/grass-gis-addons/m.analyse.trees/r.trees.mltrain/r.trees.mltrain.html +++ b/grass-gis-addons/m.analyse.trees/r.trees.mltrain/r.trees.mltrain.html @@ -22,15 +22,10 @@

Generation of training data for tree and non-tree classification

ndgb_raster=top.ndgb \ ndsm=ndsm \ slope=ndsm_slope \ - nearest=trees_nearest \ - peaks=trees_peaks \ + trees_pixel_ndvi=trees_pixel_ndvi \ + trees_raw_rast=trees_raw_rast \ group=ml_input \ save_model=ml_trees_randomforest.gz \ - ndvi_threshold=130 \ - nir_threshold=130 \ - ndsm_threshold=1 \ - slopep75_threshold=70 \ - area_threshold=5

SEE ALSO

@@ -44,4 +39,7 @@

SEE ALSO

AUTHOR

-Markus Metz, mundialis, metz at mundialis.de +

Markus Metz, mundialis, metz at mundialis.de

+

Lina Krisztian, mundialis, krisztian at mundialis.de

+

Guido Riembauer, mundialis, riembauer at mundialis.de

+

Victoria-Leandra Brunn, mundialis, brunn at mundialis.de

diff --git a/grass-gis-addons/m.analyse.trees/r.trees.mltrain/r.trees.mltrain.py b/grass-gis-addons/m.analyse.trees/r.trees.mltrain/r.trees.mltrain.py index 521c12d..88ea0d1 100644 --- a/grass-gis-addons/m.analyse.trees/r.trees.mltrain/r.trees.mltrain.py +++ b/grass-gis-addons/m.analyse.trees/r.trees.mltrain/r.trees.mltrain.py @@ -4,7 +4,7 @@ # # MODULE: r.trees.mltrain # -# AUTHOR(S): Markus Metz, Lina Krisztian +# AUTHOR(S): Markus Metz, Lina Krisztian, Guido Riembauer, Victoria-Leandra Brunn # # PURPOSE: Trains a random forest model for tree detection # @@ -53,6 +53,13 @@ # % guisection: Input # %end +# %option G_OPT_R_INPUT +# % key: trees_pixel_ndvi +# % label: raster with trees identified by NDVI value +# % answer: trees_pixel_ndvi +# % guisection: Input # ---> ???? +# %end + # %option G_OPT_R_INPUT # % key: ndvi_raster # % label: Name of the NDVI raster @@ -74,20 +81,6 @@ # % guisection: Input # %end -# %option G_OPT_R_INPUT -# % key: nearest -# % label: Name of raster with nearest peak IDs -# % answer: nearest_tree -# % guisection: Input -# %end - -# %option G_OPT_R_INPUT -# % key: peaks -# % label: Name of raster with peaks and ridges -# % answer: tree_peaks -# % guisection: Input -# %end - # %option G_OPT_R_INPUT # % key: ndwi_raster # % required: no @@ -102,49 +95,18 @@ # % guisection: Optional input # %end -# %option -# % key: ndvi_threshold -# % type: double -# % required: yes -# % label: NDVI threshold for potential trees -# % answer: 130 -# % guisection: Parameters -# %end - -# %option -# % key: nir_threshold -# % type: double -# % required: yes -# % label: NIR threshold for potential trees -# % answer: 130 -# % guisection: Parameters -# %end - -# %option -# % key: ndsm_threshold -# % type: double -# % required: yes -# % label: nDSM threshold for potential trees -# % answer: 1 -# % guisection: Parameters -# %end - -# %option -# % key: slopep75_threshold -# % type: double -# % required: yes -# % label: Threshold for 75 percentile of slope for potential trees -# % answer: 70 -# % guisection: Parameters +# %option G_OPT_R_INPUT +# % key: trees_raw_r +# % required: no +# % label: Name of the preliminary tree map raster +# % guisection: Input # %end -# %option -# % key: area_threshold -# % type: double -# % required: yes -# % label: Area size threshold for potential trees -# % answer: 5 -# % guisection: Parameters +# %option G_OPT_V_INPUT +# % key: trees_raw_v +# % required: no +# % label: Name of the preliminary tree map vector +# % guisection: Input # %end # %option @@ -175,6 +137,10 @@ # % guisection: Parallel processing # %end +# %rules +# % exclusive: trees_raw_r,trees_raw_v +# % required: trees_raw_r,trees_raw_v +# %end import atexit import os @@ -228,21 +194,29 @@ def main(): blue = options["blue_raster"] nir = options["nir_raster"] ndvi = options["ndvi_raster"] - ndvi_split = ndvi.split("@")[0] ndwi = options["ndwi_raster"] ndgb = options["ndgb_raster"] ndsm = options["ndsm"] slope = options["slope"] - nearest = options["nearest"] - peaks = options["peaks"] group_name = options["group"] model_file = options["save_model"] - ndvi_threshold = options["ndvi_threshold"] - nir_threshold = options["nir_threshold"] - ndsm_threshold = options["ndsm_threshold"] - slopep75_threshold = options["slopep75_threshold"] - area_threshold = options["area_threshold"] nprocs = int(options["nprocs"]) + trees_pixel_ndvi = options["trees_pixel_ndvi"] + + if options["trees_raw_v"]: + trees_raw_v_rast = f"trees_raw_v_rast_{os.getpid()}" + rm_rasters.append(trees_raw_v_rast) + + grass.run_command( + "v.to.rast", + input=options["trees_raw_v"], + output=trees_raw_v_rast, + use="value", + value=2, + ) + trees_basemap = trees_raw_v_rast + else: + trees_basemap = options["trees_raw_r"] nprocs = set_nprocs(nprocs) memmb = test_memory(options["memory"]) @@ -266,249 +240,16 @@ def main(): f"{ndgb} = round(127.5 * (1.0 + float({green} - {blue}) / float({green} + {blue})))" ) - # estimate trees from nearest peak IDs and various bands - - # pixel-based refinement - - # cut to ndvi - # threshold=130 - # this threshold is difficult: - # - higher such that shadow areas are removed -> many trees are removed - # - lower such that trees are kept -> shadow areas are kept - - # mathematical morphology: opening to remove isolated small patches of high ndvi - grass.run_command( - "r.neighbors", - input=ndvi, - output=f"{ndvi_split}_min1", - size=3, - method="minimum", - nprocs=nprocs, - memory=memory_max100mb, - ) - grass.run_command( - "r.neighbors", - input=f"{ndvi_split}_min1", - output=f"{ndvi_split}_min2", - size=3, - method="minimum", - nprocs=nprocs, - memory=memory_max100mb, - ) - grass.run_command( - "r.neighbors", - input=f"{ndvi_split}_min2", - output=f"{ndvi_split}_max1", - size=3, - method="maximum", - nprocs=nprocs, - memory=memory_max100mb, - ) - grass.run_command( - "r.neighbors", - input=f"{ndvi_split}_max1", - output=f"{ndvi_split}_max2", - size=3, - method="maximum", - nprocs=nprocs, - memory=memory_max100mb, - ) - rm_rasters.append(f"{ndvi_split}_min1") - rm_rasters.append(f"{ndvi_split}_min2") - rm_rasters.append(f"{ndvi_split}_max1") - rm_rasters.append(f"{ndvi_split}_max2") - - grass.mapcalc( - f"trees_pixel_ndvi = if({ndvi_split}_max2 < {ndvi_threshold}, null(), {nearest})" - ) - rm_rasters.append("trees_pixel_ndvi") - - # cut to nir: all pixels below 100 are not vegetation - # removes shadows with high ndvi e.g. on roofs - # needed - grass.mapcalc( - f"trees_pixel_nir = if({nir} < {nir_threshold}, null(), trees_pixel_ndvi)" - ) - rm_rasters.append("trees_pixel_nir") - - # cut to ndsm: all pixels below 1 meter are not tree crowns - # needed - grass.mapcalc( - f"trees_pixel_ndsm = if({ndsm} < {ndsm_threshold}, null(), trees_pixel_nir)" - ) - rm_rasters.append("trees_pixel_ndsm") - - # r.clump not diagonal again - grass.run_command( - "r.clump", input="trees_pixel_ndsm", output="trees_pixel_ndsm_unique" - ) - rm_rasters.append("trees_pixel_ndsm_unique") - - # extract peak (2), ridge (3), other (4) - grass.mapcalc( - f"trees_peak_ridge_other = if(isnull({peaks}), 4, if({peaks} == 2 || {peaks} == 3, {peaks}, 4))" - ) - rm_rasters.append("trees_peak_ridge_other") - - # remove all clumps without a peak or ridge - grass.run_command( - "r.stats.zonal", - base="trees_pixel_ndsm_unique", - cover="trees_peak_ridge_other", - output="trees_pixel_ndsm_unique_min", - method="min", - ) - rm_rasters.append("trees_pixel_ndsm_unique_min") - grass.mapcalc( - "trees_pixel_ndsm_unique_filt = if(trees_pixel_ndsm_unique_min > 3, null(), trees_pixel_ndsm_unique)" - ) - rm_rasters.append("trees_pixel_ndsm_unique_filt") - - # fill gaps after pixel-based refinement - # mathematical morphology: dilation - grass.run_command( - "r.neighbors", - input="trees_pixel_ndsm_unique_filt", - output="trees_pixel_filt_fill1_dbl", - size=3, - method="mode", - nprocs=nprocs, - memory=memory_max100mb, - ) - grass.mapcalc("trees_pixel_filt_fill1 = round(trees_pixel_filt_fill1_dbl)") - # remove large DCELL map immediately - grass.run_command( - "g.remove", type="raster", name="trees_pixel_filt_fill1_dbl", flags="f" - ) - rm_rasters.append("trees_pixel_filt_fill1") - grass.run_command( - "r.neighbors", - input="trees_pixel_filt_fill1", - output="trees_pixel_filt_fill2_dbl", - size=3, - method="mode", - nprocs=nprocs, - memory=memory_max100mb, - ) - grass.mapcalc("trees_pixel_filt_fill2 = round(trees_pixel_filt_fill2_dbl)") - # remove large DCELL map immediately - grass.run_command( - "g.remove", type="raster", name="trees_pixel_filt_fill2_dbl", flags="f" - ) - rm_rasters.append("trees_pixel_filt_fill2") - - # create new clumps - # r.clump not diagonal - grass.run_command( - "r.clump", input="trees_pixel_filt_fill2", output="trees_object_all" - ) - rm_rasters.append("trees_object_all") - - # object-based refinement - - # remove low-lying objects with max(ndsm) < 3 - # needed - grass.run_command( - "r.stats.zonal", - base="trees_object_all", - cover=ndsm, - method="max", - output="trees_object_ndsmmax", - ) - rm_rasters.append("trees_object_ndsmmax") - grass.mapcalc( - f"trees_object_ndsm = if(trees_object_ndsmmax < {ndsm_threshold}, null(), trees_object_all)" - ) - rm_rasters.append("trees_object_ndsm") - - # mean NDVI per object must be > X ? - # some effect - grass.run_command( - "r.stats.zonal", - base="trees_object_ndsm", - cover=ndvi, - method="average", - output="trees_object_ndviavg", - ) - rm_rasters.append("trees_object_ndviavg") - grass.mapcalc( - f"trees_object_ndvi = if(trees_object_ndviavg < {ndvi_threshold}, null(), trees_object_all)" - ) - rm_rasters.append("trees_object_ndvi") - - # problems - # roofs with some vegetation - # solar panels - - # normalized difference green-blue - # for solar panels - # threshold 121 removes also some trees (dark trees, trees partially shadowed by other trees) - # r.stats.zonal base=trees_object_all cover=TOM_378000_5711000_20cm.ndgb method=average output=trees_object_ndgb - - # green: not specific enough - - # slope - # removes bushes with a height of 3-5 meter - # needed - # r.stats.zonal base=trees_object_all cover=ndsm_slope method=average output=trees_object_slope_avg - grass.run_command( - "r.stats.quantile", - base="trees_object_ndvi", - cover=slope, - percentiles="75,90", - output="trees_object_slope_p75,trees_object_slope_p90", - ) - rm_rasters.append("trees_object_slope_p75") - rm_rasters.append("trees_object_slope_p90") - - # threshold for slope_p75: 70 - grass.mapcalc( - f"trees_object_slope = if(trees_object_slope_p75 < {slopep75_threshold}, null(), trees_object_ndvi)" - ) - rm_rasters.append("trees_object_slope") - - # vectorize - grass.run_command( - "r.to.vect", - input="trees_object_slope", - output="trees_object_filt_all", - type="area", - flags="sv", - ) - rm_vectors.append("trees_object_filt_all") - - # remove small areas smaller than 5sqm - grass.run_command( - "v.clean", - input="trees_object_filt_all", - output="trees_object_filt_large", - tool="rmarea", - threshold=area_threshold, - ) - rm_vectors.append("trees_object_filt_large") - - # rasterize again - grass.run_command( - "v.to.rast", - input="trees_object_filt_large", - output="trees_object_filt_large", - type="area", - use="cat", - ) - rm_rasters.append("trees_object_filt_large") - # extract training points - # trees: trees_object_filt_large - grass.mapcalc("trees_bin = if(isnull(trees_object_filt_large), null(), 2)") # extract 4000 cells grass.run_command( "r.random", - input="trees_bin", + input=trees_basemap, raster="trees_trainpnts", npoints=4000, flags="s", ) - rm_rasters.append("trees_bin") + rm_rasters.append("trees_trainpnts") # non trees @@ -516,8 +257,9 @@ def main(): # false trees # problem areas with high NDVI like shadows on roofs, solar panels # trees_object_filt_large = NULL and trees_pixel_ndvi != NULL + rm_rasters.append(trees_pixel_ndvi) grass.mapcalc( - "false_trees = if(isnull(trees_pixel_ndvi), null(), if(isnull(trees_object_filt_large), 1, null()))" + f"false_trees = if(isnull({trees_pixel_ndvi}), null(), if(isnull({trees_basemap}), 1, null()))" ) grass.run_command( "r.random", @@ -531,7 +273,7 @@ def main(): # other areas clearly not trees grass.mapcalc( - "notrees = if(isnull(trees_pixel_ndvi) && isnull(trees_object_filt_large), 1, null())" + f"notrees = if(isnull({trees_pixel_ndvi}) && isnull({trees_basemap}), 1, null())" ) grass.run_command( "r.random", diff --git a/grass-gis-addons/m.analyse.trees/r.trees.traindata/Makefile b/grass-gis-addons/m.analyse.trees/r.trees.traindata/Makefile new file mode 100644 index 0000000..8272ec0 --- /dev/null +++ b/grass-gis-addons/m.analyse.trees/r.trees.traindata/Makefile @@ -0,0 +1,7 @@ +MODULE_TOPDIR = ../../.. + +PGM = r.trees.traindata + +include $(MODULE_TOPDIR)/include/Make/Script.make + +default: script diff --git a/grass-gis-addons/m.analyse.trees/r.trees.traindata/r.trees.traindata.html b/grass-gis-addons/m.analyse.trees/r.trees.traindata/r.trees.traindata.html new file mode 100644 index 0000000..22f8482 --- /dev/null +++ b/grass-gis-addons/m.analyse.trees/r.trees.traindata/r.trees.traindata.html @@ -0,0 +1,43 @@ +

DESCRIPTION

+ +r.trees.traindata generates a preliminary tree map for random forest classification and/or post processing. + +

This module requires that pixels have been assigned to the nearest +potential tree peak with r.trees.peaks.

+ +

EXAMPLES

+ +

Generation of training data for tree and non-tree classification

+ +
+r.trees.traindata green_raster=top.green \
+                  blue_raster=top.blue \
+                  nir_raster=top.nir \
+                  ndvi_raster=top.ndvi \
+                  ndwi_raster=top.ndwi \
+                  ndgb_raster=top.ndgb \
+                  ndsm=ndsm \
+                  slope=ndsm_slope \
+                  nearest=trees_nearest \
+                  peaks=trees_peaks \
+                  traindata_r=trees_raw_rast \
+                  ndvi_threshold=130 \
+                  nir_threshold=130 \
+                  ndsm_threshold=1 \
+                  slopep75_threshold=70 \
+                  area_threshold=5 
+
+ +

SEE ALSO

+ + +m.analyse.trees, +r.geomorphon, + + +

AUTHOR

+ +

Markus Metz, mundialis, metz at mundialis.de

+

Lina Krisztian, mundialis, krisztian at mundialis.de

+

Guido Riembauer, mundialis, riembauer at mundialis.de

+

Victoria-Leandra Brunn, mundialis, brunn at mundialis.de

\ No newline at end of file diff --git a/grass-gis-addons/m.analyse.trees/r.trees.traindata/r.trees.traindata.py b/grass-gis-addons/m.analyse.trees/r.trees.traindata/r.trees.traindata.py new file mode 100644 index 0000000..caee502 --- /dev/null +++ b/grass-gis-addons/m.analyse.trees/r.trees.traindata/r.trees.traindata.py @@ -0,0 +1,507 @@ +#!/usr/bin/env python3 + +############################################################################ +# +# MODULE: r.trees.traindata +# +# AUTHOR(S): Markus Metz, Lina Krisztian, Guido Riembauer, +# Victoria-Leandra Brunn +# +# PURPOSE: Creates a preliminary tree map for random forest classification +# and/or post processing. +# +# COPYRIGHT: (C) 2023 - 2024 by mundialis and the GRASS Development Team +# +# This program is free software under the GNU General Public +# License (>=v2). Read the file COPYING that comes with GRASS +# for details. +# +############################################################################# + +# %Module +# % description: Creates a preliminary tree map for random forest classification and/or post processing. +# % keyword: raster +# % keyword: statistics +# % keyword: machine learning +# % keyword: trees analysis +# %end + +# %option G_OPT_R_INPUT +# % key: green_raster +# % label: Name of the green raster +# % answer: top_green_02 +# % guisection: Input +# %end + +# %option G_OPT_R_INPUT +# % key: blue_raster +# % label: Name of the blue raster +# % answer: top_blue_02 +# % guisection: Input +# %end + +# %option G_OPT_R_INPUT +# % key: nir_raster +# % label: Name of the NIR raster +# % answer: top_nir_02 +# % guisection: Input +# %end + +# %option G_OPT_R_INPUT +# % key: ndvi_raster +# % label: Name of the NDVI raster +# % answer: top_ndvi_02 +# % guisection: Input +# %end + +# %option G_OPT_R_INPUT +# % key: ndsm +# % label: Name of the nDSM raster +# % answer: ndsm +# % guisection: Input +# %end + +# %option G_OPT_R_INPUT +# % key: slope +# % label: Name of the nDSM slope raster +# % answer: ndsm_slope +# % guisection: Input +# %end + +# %option G_OPT_R_INPUT +# % key: nearest +# % label: Name of raster with nearest peak IDs +# % answer: nearest_tree +# % guisection: Input +# %end + +# %option G_OPT_R_INPUT +# % key: peaks +# % label: Name of raster with peaks and ridges +# % answer: tree_peaks +# % guisection: Input +# %end + +# %option G_OPT_R_INPUT +# % key: ndwi_raster +# % required: no +# % label: Name of the NDWI raster +# % guisection: Optional input +# %end + +# %option G_OPT_R_INPUT +# % key: ndgb_raster +# % required: no +# % label: Name of the normalized green-blue difference raster +# % guisection: Optional input +# %end + +# %option +# % key: ndvi_threshold +# % type: double +# % required: yes +# % label: NDVI threshold for potential trees +# % answer: 130 +# % guisection: Parameters +# %end + +# %option +# % key: nir_threshold +# % type: double +# % required: yes +# % label: NIR threshold for potential trees +# % answer: 130 +# % guisection: Parameters +# %end + +# %option +# % key: ndsm_threshold +# % type: double +# % required: yes +# % label: nDSM threshold for potential trees +# % answer: 1 +# % guisection: Parameters +# %end + +# %option +# % key: slopep75_threshold +# % type: double +# % required: yes +# % label: Threshold for 75 percentile of slope for potential trees +# % answer: 70 +# % guisection: Parameters +# %end + +# %option +# % key: area_threshold +# % type: double +# % required: yes +# % label: Area size threshold for potential trees +# % answer: 5 +# % guisection: Parameters +# %end + +# %option G_OPT_R_OUTPUT +# % key: traindata_r +# % label: primary tree map as raster +# % description: Primary tree map in raster format to pass to mltrain +# % answer: traindata_r +# % guisection: Output +# %end + +# %option G_OPT_V_OUTPUT +# % key: traindata_v +# % label: primary tree map as vector +# % description: Primary tree map in vector format to export and edit +# % answer: traindata_v +# % guisection: Output +# %end + +# %option G_OPT_MEMORYMB +# % guisection: Parallel processing +# %end + +# %option G_OPT_M_NPROCS +# % label: Number of cores for multiprocessing, -2 is the number of available cores - 1 +# % answer: -2 +# % guisection: Parallel processing +# %end + +# %rules +# % exclusive: traindata_r,traindata_v +# % required: traindata_r,traindata_v +# %end + +import atexit +import os +import sys +import grass.script as grass +from grass.pygrass.utils import get_lib_path + +# initialize global vars +rm_rasters = [] +rm_vectors = [] +rm_groups = [] +tmp_mask_old = None + + +def cleanup(): + nuldev = open(os.devnull, "w") + kwargs = {"flags": "f", "quiet": True, "stderr": nuldev} + for rmrast in rm_rasters: + if grass.find_file(name=rmrast, element="cell")["file"]: + grass.run_command("g.remove", type="raster", name=rmrast, **kwargs) + for rmv in rm_vectors: + if grass.find_file(name=rmv, element="vector")["file"]: + grass.run_command("g.remove", type="vector", name=rmv, **kwargs) + for rmgroup in rm_groups: + if grass.find_file(name=rmgroup, element="group")["file"]: + grass.run_command("g.remove", type="group", name=rmgroup, **kwargs) + grass.del_temp_region() + + +def main(): + global rm_rasters, tmp_mask_old, rm_vectors, rm_groups + + path = get_lib_path(modname="m.analyse.trees", libname="analyse_trees_lib") + if path is None: + grass.fatal("Unable to find the analyse trees library directory") + sys.path.append(path) + try: + from analyse_trees_lib import set_nprocs, test_memory + except Exception: + grass.fatal("m.analyse.trees library is not installed") + + grass.message(_("Preparing input data...")) + if grass.find_file(name="MASK", element="cell")["file"]: + tmp_mask_old = "tmp_mask_old_%s" % os.getpid() + grass.run_command( + "g.rename", raster="%s,%s" % ("MASK", tmp_mask_old), quiet=True + ) + + green = options["green_raster"] + blue = options["blue_raster"] + nir = options["nir_raster"] + ndvi = options["ndvi_raster"] + ndvi_split = ndvi.split("@")[0] + ndwi = options["ndwi_raster"] + ndgb = options["ndgb_raster"] + ndsm = options["ndsm"] + slope = options["slope"] + nearest = options["nearest"] + peaks = options["peaks"] + ndvi_threshold = options["ndvi_threshold"] + nir_threshold = options["nir_threshold"] + ndsm_threshold = options["ndsm_threshold"] + slopep75_threshold = options["slopep75_threshold"] + area_threshold = options["area_threshold"] + nprocs = int(options["nprocs"]) + + nprocs = set_nprocs(nprocs) + memmb = test_memory(options["memory"]) + # for some modules like r.neighbors and r.slope_aspect, there is + # no speed gain by using more than 100 MB RAM + memory_max100mb = 100 + if memmb < 100: + memory_max100mb = memmb + + grass.use_temp_region() + + if not ndwi: + ndwi = "ndwi" + grass.mapcalc( + f"{ndwi} = round(127.5 * (1.0 + float({green} - {nir}) / float({green} + {nir})))" + ) + + if not ndgb: + ndgb = "ndgb" + grass.mapcalc( + f"{ndgb} = round(127.5 * (1.0 + float({green} - {blue}) / float({green} + {blue})))" + ) + + # estimate trees from nearest peak IDs and various bands + + # pixel-based refinement + + # cut to ndvi + # threshold=130 + # this threshold is difficult: + # - higher such that shadow areas are removed -> many trees are removed + # - lower such that trees are kept -> shadow areas are kept + + # mathematical morphology: opening to remove isolated small patches of high ndvi + grass.run_command( + "r.neighbors", + input=ndvi, + output=f"{ndvi_split}_min1", + size=3, + method="minimum", + nprocs=nprocs, + memory=memory_max100mb, + ) + grass.run_command( + "r.neighbors", + input=f"{ndvi_split}_min1", + output=f"{ndvi_split}_min2", + size=3, + method="minimum", + nprocs=nprocs, + memory=memory_max100mb, + ) + grass.run_command( + "r.neighbors", + input=f"{ndvi_split}_min2", + output=f"{ndvi_split}_max1", + size=3, + method="maximum", + nprocs=nprocs, + memory=memory_max100mb, + ) + grass.run_command( + "r.neighbors", + input=f"{ndvi_split}_max1", + output=f"{ndvi_split}_max2", + size=3, + method="maximum", + nprocs=nprocs, + memory=memory_max100mb, + ) + rm_rasters.append(f"{ndvi_split}_min1") + rm_rasters.append(f"{ndvi_split}_min2") + rm_rasters.append(f"{ndvi_split}_max1") + rm_rasters.append(f"{ndvi_split}_max2") + + grass.mapcalc( + f"trees_pixel_ndvi = if({ndvi_split}_max2 < {ndvi_threshold}, null(), {nearest})" + ) + # rm_rasters.append("trees_pixel_ndvi") --> keep raster for mltrain + + # cut to nir: all pixels below 100 are not vegetation + # removes shadows with high ndvi e.g. on roofs + # needed + grass.mapcalc( + f"trees_pixel_nir = if({nir} < {nir_threshold}, null(), trees_pixel_ndvi)" + ) + rm_rasters.append("trees_pixel_nir") + + # cut to ndsm: all pixels below 1 meter are not tree crowns + # needed + grass.mapcalc( + f"trees_pixel_ndsm = if({ndsm} < {ndsm_threshold}, null(), trees_pixel_nir)" + ) + rm_rasters.append("trees_pixel_ndsm") + + # r.clump not diagonal again + grass.run_command( + "r.clump", input="trees_pixel_ndsm", output="trees_pixel_ndsm_unique" + ) + rm_rasters.append("trees_pixel_ndsm_unique") + + # extract peak (2), ridge (3), other (4) + grass.mapcalc( + f"trees_peak_ridge_other = if(isnull({peaks}), 4, if({peaks} == 2 || {peaks} == 3, {peaks}, 4))" + ) + rm_rasters.append("trees_peak_ridge_other") + + # remove all clumps without a peak or ridge + grass.run_command( + "r.stats.zonal", + base="trees_pixel_ndsm_unique", + cover="trees_peak_ridge_other", + output="trees_pixel_ndsm_unique_min", + method="min", + ) + rm_rasters.append("trees_pixel_ndsm_unique_min") + grass.mapcalc( + "trees_pixel_ndsm_unique_filt = if(trees_pixel_ndsm_unique_min > 3, null(), trees_pixel_ndsm_unique)" + ) + rm_rasters.append("trees_pixel_ndsm_unique_filt") + + # fill gaps after pixel-based refinement + # mathematical morphology: dilation + grass.run_command( + "r.neighbors", + input="trees_pixel_ndsm_unique_filt", + output="trees_pixel_filt_fill1_dbl", + size=3, + method="mode", + nprocs=nprocs, + memory=memory_max100mb, + ) + grass.mapcalc("trees_pixel_filt_fill1 = round(trees_pixel_filt_fill1_dbl)") + # remove large DCELL map immediately + grass.run_command( + "g.remove", type="raster", name="trees_pixel_filt_fill1_dbl", flags="f" + ) + rm_rasters.append("trees_pixel_filt_fill1") + grass.run_command( + "r.neighbors", + input="trees_pixel_filt_fill1", + output="trees_pixel_filt_fill2_dbl", + size=3, + method="mode", + nprocs=nprocs, + memory=memory_max100mb, + ) + grass.mapcalc("trees_pixel_filt_fill2 = round(trees_pixel_filt_fill2_dbl)") + # remove large DCELL map immediately + grass.run_command( + "g.remove", type="raster", name="trees_pixel_filt_fill2_dbl", flags="f" + ) + rm_rasters.append("trees_pixel_filt_fill2") + + # create new clumps + # r.clump not diagonal + grass.run_command( + "r.clump", input="trees_pixel_filt_fill2", output="trees_object_all" + ) + rm_rasters.append("trees_object_all") + + # object-based refinement + + # remove low-lying objects with max(ndsm) < 3 + # needed + grass.run_command( + "r.stats.zonal", + base="trees_object_all", + cover=ndsm, + method="max", + output="trees_object_ndsmmax", + ) + rm_rasters.append("trees_object_ndsmmax") + grass.mapcalc( + f"trees_object_ndsm = if(trees_object_ndsmmax < {ndsm_threshold}, null(), trees_object_all)" + ) + rm_rasters.append("trees_object_ndsm") + + # mean NDVI per object must be > X ? + # some effect + grass.run_command( + "r.stats.zonal", + base="trees_object_ndsm", + cover=ndvi, + method="average", + output="trees_object_ndviavg", + ) + rm_rasters.append("trees_object_ndviavg") + grass.mapcalc( + f"trees_object_ndvi = if(trees_object_ndviavg < {ndvi_threshold}, null(), trees_object_all)" + ) + rm_rasters.append("trees_object_ndvi") + + # problems + # roofs with some vegetation + # solar panels + + # normalized difference green-blue + # for solar panels + # threshold 121 removes also some trees (dark trees, trees partially shadowed by other trees) + # r.stats.zonal base=trees_object_all cover=TOM_378000_5711000_20cm.ndgb method=average output=trees_object_ndgb + + # green: not specific enough + + # slope + # removes bushes with a height of 3-5 meter + # needed + # r.stats.zonal base=trees_object_all cover=ndsm_slope method=average output=trees_object_slope_avg + grass.run_command( + "r.stats.quantile", + base="trees_object_ndvi", + cover=slope, + percentiles="75,90", + output="trees_object_slope_p75,trees_object_slope_p90", + ) + rm_rasters.append("trees_object_slope_p75") + rm_rasters.append("trees_object_slope_p90") + + # threshold for slope_p75: 70 + grass.mapcalc( + f"trees_object_slope = if(trees_object_slope_p75 < {slopep75_threshold}, null(), trees_object_ndvi)" + ) + rm_rasters.append("trees_object_slope") + + # vectorize + grass.run_command( + "r.to.vect", + input="trees_object_slope", + output="trees_object_filt_all", + type="area", + flags="sv", + ) + rm_vectors.append("trees_object_filt_all") + + # remove small areas smaller than 5sqm + grass.run_command( + "v.clean", + input="trees_object_filt_all", + output="trees_object_filt_large", + tool="rmarea", + threshold=area_threshold, + ) + + # rasterize again + grass.run_command( + "v.to.rast", + input="trees_object_filt_large", + output="trees_object_filt_large", + type="area", + use="cat", + ) + rm_rasters.append("trees_object_filt_large") + + # trees: trees_object_filt_large + if options["traindata_r"]: + grass.mapcalc(f"{options["traindata_r"]} = if(isnull(trees_object_filt_large), null(), 2)") + rm_vectors.append("trees_object_filt_large") + + elif options["traindata_v"]: + grass.run_command( + "g.rename", + vector=f"trees_object_filt_large,{options['traindata_v']}" + ) + + +if __name__ == "__main__": + options, flags = grass.parser() + atexit.register(cleanup) + main() From d1b12244f7f2c14047486c3fbccbe906e391cc23 Mon Sep 17 00:00:00 2001 From: vbrunn Date: Fri, 18 Oct 2024 15:53:06 +0200 Subject: [PATCH 02/11] complete split edits (input params, num_pixels, output) --- .../r.trees.mltrain/r.trees.mltrain.html | 3 +- .../r.trees.mltrain/r.trees.mltrain.py | 144 ++++++++++-------- .../r.trees.traindata/r.trees.traindata.html | 3 +- .../r.trees.traindata/r.trees.traindata.py | 38 +++-- 4 files changed, 105 insertions(+), 83 deletions(-) diff --git a/grass-gis-addons/m.analyse.trees/r.trees.mltrain/r.trees.mltrain.html b/grass-gis-addons/m.analyse.trees/r.trees.mltrain/r.trees.mltrain.html index df331a1..4326a05 100644 --- a/grass-gis-addons/m.analyse.trees/r.trees.mltrain/r.trees.mltrain.html +++ b/grass-gis-addons/m.analyse.trees/r.trees.mltrain/r.trees.mltrain.html @@ -25,14 +25,13 @@

Generation of training data for tree and non-tree classification

trees_pixel_ndvi=trees_pixel_ndvi \ trees_raw_rast=trees_raw_rast \ group=ml_input \ - save_model=ml_trees_randomforest.gz \ + save_model=ml_trees_randomforest.gz

SEE ALSO

m.analyse.trees, -r.geomorphon, r.learn.train, r.learn.predict diff --git a/grass-gis-addons/m.analyse.trees/r.trees.mltrain/r.trees.mltrain.py b/grass-gis-addons/m.analyse.trees/r.trees.mltrain/r.trees.mltrain.py index 88ea0d1..3345611 100644 --- a/grass-gis-addons/m.analyse.trees/r.trees.mltrain/r.trees.mltrain.py +++ b/grass-gis-addons/m.analyse.trees/r.trees.mltrain/r.trees.mltrain.py @@ -57,7 +57,7 @@ # % key: trees_pixel_ndvi # % label: raster with trees identified by NDVI value # % answer: trees_pixel_ndvi -# % guisection: Input # ---> ???? +# % guisection: Input # %end # %option G_OPT_R_INPUT @@ -109,6 +109,14 @@ # % guisection: Input # %end +# %option +# % key: num_samples +# % type: integer +# % required: no +# % label: Number of sample points for each class in training +# % guisection: Optional input +# %end + # %option # % key: group # % type: string @@ -127,10 +135,6 @@ # % guisection: Output # %end -# %option G_OPT_MEMORYMB -# % guisection: Parallel processing -# %end - # %option G_OPT_M_NPROCS # % label: Number of cores for multiprocessing, -2 is the number of available cores - 1 # % answer: -2 @@ -150,8 +154,6 @@ # initialize global vars rm_rasters = [] -rm_vectors = [] -rm_groups = [] tmp_mask_old = None @@ -161,24 +163,28 @@ def cleanup(): for rmrast in rm_rasters: if grass.find_file(name=rmrast, element="cell")["file"]: grass.run_command("g.remove", type="raster", name=rmrast, **kwargs) - for rmv in rm_vectors: - if grass.find_file(name=rmv, element="vector")["file"]: - grass.run_command("g.remove", type="vector", name=rmv, **kwargs) - for rmgroup in rm_groups: - if grass.find_file(name=rmgroup, element="group")["file"]: - grass.run_command("g.remove", type="group", name=rmgroup, **kwargs) grass.del_temp_region() +def numsamplecheck(number_samples, rastername): + raster_samples = int(grass.parse_command("r.univar", map=rastername, flags="g")["n"]) + if number_samples > raster_samples: + grass.warning( + _(f"The chosen number of pixels {number_samples} is exceeding the total number of ", + f"non-null pixels in the given rastermap {rastername}. ", + f"The number of pixels will be set to the maximal amount of {raster_samples}.") + ) + number_samples = raster_samples + return number_samples def main(): - global rm_rasters, tmp_mask_old, rm_vectors, rm_groups + global rm_rasters, tmp_mask_old path = get_lib_path(modname="m.analyse.trees", libname="analyse_trees_lib") if path is None: grass.fatal("Unable to find the analyse trees library directory") sys.path.append(path) try: - from analyse_trees_lib import set_nprocs, test_memory + from analyse_trees_lib import set_nprocs except Exception: grass.fatal("m.analyse.trees library is not installed") @@ -203,28 +209,7 @@ def main(): nprocs = int(options["nprocs"]) trees_pixel_ndvi = options["trees_pixel_ndvi"] - if options["trees_raw_v"]: - trees_raw_v_rast = f"trees_raw_v_rast_{os.getpid()}" - rm_rasters.append(trees_raw_v_rast) - - grass.run_command( - "v.to.rast", - input=options["trees_raw_v"], - output=trees_raw_v_rast, - use="value", - value=2, - ) - trees_basemap = trees_raw_v_rast - else: - trees_basemap = options["trees_raw_r"] - nprocs = set_nprocs(nprocs) - memmb = test_memory(options["memory"]) - # for some modules like r.neighbors and r.slope_aspect, there is - # no speed gain by using more than 100 MB RAM - memory_max100mb = 100 - if memmb < 100: - memory_max100mb = memmb grass.use_temp_region() @@ -240,55 +225,82 @@ def main(): f"{ndgb} = round(127.5 * (1.0 + float({green} - {blue}) / float({green} + {blue})))" ) - # extract training points - # extract 4000 cells - grass.run_command( - "r.random", - input=trees_basemap, - raster="trees_trainpnts", - npoints=4000, - flags="s", - ) - - rm_rasters.append("trees_trainpnts") + if options["trees_raw_v"]: + trees_raw_v_rast = f"trees_raw_v_rast_{os.getpid()}" + rm_rasters.append(trees_raw_v_rast) + + grass.run_command( + "v.to.rast", + input=options["trees_raw_v"], + output=trees_raw_v_rast, + use="value", + value=2, + ) + trees_basemap = trees_raw_v_rast + else: + trees_basemap = options["trees_raw_r"] # non trees # false trees # problem areas with high NDVI like shadows on roofs, solar panels # trees_object_filt_large = NULL and trees_pixel_ndvi != NULL - rm_rasters.append(trees_pixel_ndvi) grass.mapcalc( f"false_trees = if(isnull({trees_pixel_ndvi}), null(), if(isnull({trees_basemap}), 1, null()))" ) - grass.run_command( - "r.random", - input="false_trees", - raster="false_trees_trainpnts", - npoints=4000, - flags="s", - ) - rm_rasters.append("false_trees") - rm_rasters.append("false_trees_trainpnts") # other areas clearly not trees grass.mapcalc( f"notrees = if(isnull({trees_pixel_ndvi}) && isnull({trees_basemap}), 1, null())" ) - grass.run_command( - "r.random", - input="notrees", - raster="notrees_trainpnts", - npoints=4000, - flags="s", - ) + rm_rasters.append("false_trees") rm_rasters.append("notrees") - rm_rasters.append("notrees_trainpnts") + + # extract training points + # extract "num_samples" cells if given + if options["num_samples"]: + num_samples = int(options["num_samples"]) + + realsamples = numsamplecheck(num_samples, trees_basemap) + grass.run_command( + "r.random", + input=trees_basemap, + raster="trees_trainpnts", + npoints=realsamples, + flags="s", + ) + + # false trees + realsamples = numsamplecheck(num_samples, "false_trees") + grass.run_command( + "r.random", + input="false_trees", + raster="false_trees_trainpnts", + npoints=realsamples, + flags="s", + ) + + # other areas clearly not trees + realsamples = numsamplecheck(num_samples, "notrees") + grass.run_command( + "r.random", + input="notrees", + raster="notrees_trainpnts", + npoints=realsamples, + flags="s", + ) + rm_rasters.append("trees_trainpnts") + rm_rasters.append("false_trees_trainpnts") + rm_rasters.append("notrees_trainpnts") + + patch_list = ["trees_trainpnts", "false_trees_trainpnts", "notrees_trainpnts"] + else: + patch_list = [trees_basemap, "false_trees", "notrees"] # patch trees, false trees and non-trees grass.run_command( "r.patch", - input="trees_trainpnts,false_trees_trainpnts,notrees_trainpnts", + input=patch_list, output="ml_trainpnts", ) rm_rasters.append("ml_trainpnts") diff --git a/grass-gis-addons/m.analyse.trees/r.trees.traindata/r.trees.traindata.html b/grass-gis-addons/m.analyse.trees/r.trees.traindata/r.trees.traindata.html index 22f8482..4f80664 100644 --- a/grass-gis-addons/m.analyse.trees/r.trees.traindata/r.trees.traindata.html +++ b/grass-gis-addons/m.analyse.trees/r.trees.traindata/r.trees.traindata.html @@ -25,7 +25,8 @@

Generation of training data for tree and non-tree classification

nir_threshold=130 \ ndsm_threshold=1 \ slopep75_threshold=70 \ - area_threshold=5 + area_threshold=5 \ + trees_pixel_ndvi=trees_pixel_ndvi

SEE ALSO

diff --git a/grass-gis-addons/m.analyse.trees/r.trees.traindata/r.trees.traindata.py b/grass-gis-addons/m.analyse.trees/r.trees.traindata/r.trees.traindata.py index caee502..f208166 100644 --- a/grass-gis-addons/m.analyse.trees/r.trees.traindata/r.trees.traindata.py +++ b/grass-gis-addons/m.analyse.trees/r.trees.traindata/r.trees.traindata.py @@ -145,7 +145,7 @@ # % key: traindata_r # % label: primary tree map as raster # % description: Primary tree map in raster format to pass to mltrain -# % answer: traindata_r +# % required: no # % guisection: Output # %end @@ -153,7 +153,15 @@ # % key: traindata_v # % label: primary tree map as vector # % description: Primary tree map in vector format to export and edit -# % answer: traindata_v +# % required: no +# % guisection: Output +# %end + +# %option G_OPT_R_OUTPUT +# % key: trees_pixel_ndvi +# % label: Name of raster with nearest peak IDs filtered by NDVI +# % description: necessary intermediate result for mltrain +# % required: no # % guisection: Output # %end @@ -168,7 +176,6 @@ # %end # %rules -# % exclusive: traindata_r,traindata_v # % required: traindata_r,traindata_v # %end @@ -181,7 +188,6 @@ # initialize global vars rm_rasters = [] rm_vectors = [] -rm_groups = [] tmp_mask_old = None @@ -194,14 +200,11 @@ def cleanup(): for rmv in rm_vectors: if grass.find_file(name=rmv, element="vector")["file"]: grass.run_command("g.remove", type="vector", name=rmv, **kwargs) - for rmgroup in rm_groups: - if grass.find_file(name=rmgroup, element="group")["file"]: - grass.run_command("g.remove", type="group", name=rmgroup, **kwargs) grass.del_temp_region() def main(): - global rm_rasters, tmp_mask_old, rm_vectors, rm_groups + global rm_rasters, tmp_mask_old, rm_vectors path = get_lib_path(modname="m.analyse.trees", libname="analyse_trees_lib") if path is None: @@ -311,16 +314,22 @@ def main(): rm_rasters.append(f"{ndvi_split}_max1") rm_rasters.append(f"{ndvi_split}_max2") + if options["trees_pixel_ndvi"]: + trees_pixel_ndvi = options["trees_pixel_ndvi"] + else: + trees_pixel_ndvi = "trees_pixel_ndvi" + rm_rasters.append(trees_pixel_ndvi) + grass.mapcalc( - f"trees_pixel_ndvi = if({ndvi_split}_max2 < {ndvi_threshold}, null(), {nearest})" + f"{trees_pixel_ndvi} = if({ndvi_split}_max2 < {ndvi_threshold}, null(), {nearest})" ) - # rm_rasters.append("trees_pixel_ndvi") --> keep raster for mltrain + # cut to nir: all pixels below 100 are not vegetation # removes shadows with high ndvi e.g. on roofs # needed grass.mapcalc( - f"trees_pixel_nir = if({nir} < {nir_threshold}, null(), trees_pixel_ndvi)" + f"trees_pixel_nir = if({nir} < {nir_threshold}, null(), {trees_pixel_ndvi})" ) rm_rasters.append("trees_pixel_nir") @@ -491,10 +500,11 @@ def main(): # trees: trees_object_filt_large if options["traindata_r"]: - grass.mapcalc(f"{options["traindata_r"]} = if(isnull(trees_object_filt_large), null(), 2)") - rm_vectors.append("trees_object_filt_large") + grass.mapcalc(f"{options['traindata_r']} = if(isnull(trees_object_filt_large), null(), 2)") + if not options["traindata_v"]: + rm_vectors.append("trees_object_filt_large") - elif options["traindata_v"]: + if options["traindata_v"]: grass.run_command( "g.rename", vector=f"trees_object_filt_large,{options['traindata_v']}" From 892941da096bf9b889e7502be9235414e9167c46 Mon Sep 17 00:00:00 2001 From: vbrunn Date: Fri, 18 Oct 2024 16:05:47 +0200 Subject: [PATCH 03/11] Notes of possible add-on combinations --- Doku_hints.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 Doku_hints.md diff --git a/Doku_hints.md b/Doku_hints.md new file mode 100644 index 0000000..b603f00 --- /dev/null +++ b/Doku_hints.md @@ -0,0 +1,23 @@ +## Dokumentations Gedankenstützen + +Änderungen nach r.trees.peaks +* Möglichkeit 1: Raster und direkt in r.trees.postprocess: +r.trees.traindata green_raster=dop_green blue_raster=dop_blue nir_raster=dop_nir ndvi_raster=ndvi_raster ndsm=ndsm slope=slope2 nearest=nearest_tree2 peaks=tree_peaks2 traindata_r=traindata_raster1 trees_pixel_ndvi=trees_pixel_ndvi1 --overwrite +r.trees.postprocess tree_pixels=traindata_raster1 green_raster=dop_green blue_raster=dop_blue nir_raster=dop_nir ndvi_raster=ndvi_raster ndsm=ndsm slope=slope2 nearest=nearest_tree2 peaks=tree_peaks2 trees_raster=tree_objects3 trees_vector=tree_objects3 --overwrite + +* Möglichkeit 2: Raster und ML (r.trees.mltrain, r.trees.mlapply, r.trees.postprocess) +r.trees.traindata green_raster=dop_green blue_raster=dop_blue nir_raster=dop_nir ndvi_raster=ndvi_raster ndsm=ndsm slope=slope2 nearest=> +r.trees.mltrain red_raster=dop_red green_raster=dop_green blue_raster=dop_blue nir_raster=dop_nir ndvi_raster=ndvi_raster ndsm=ndsm slope=slope trees_pixel_ndvi=trees_pixel_ndvi1 trees_raw_r=traindata_raster1 num_samples=8000 group=ml_input_test save_model=test_ml_trees_randomforest.gz --o +r.trees.mlapply area=aoi group=ml_input_test model=test_ml_trees_randomforest.gz output=tree_pixels4 --overwrite +r.trees.postprocess tree_pixels=tree_pixels4 green_raster=dop_green blue_raster=dop_blue nir_raster=dop_nir ndvi_raster=ndvi_raster ndsm=ndsm slope=slope2 nearest=nearest_tree2 peaks=tree_peaks2 trees_raster=tree_objects4 trees_vector=tree_objects4 --overwrite + + +* Möglichkeit 3: Vektor und ML (r.trees.mltrain, r.trees.mlapply, r.trees.postprocess) +r.trees.traindata green_raster=dop_green blue_raster=dop_blue nir_raster=dop_nir ndvi_raster=ndvi_raster ndsm=ndsm slope=slope2 nearest=nearest_tree2 peaks=tree_peaks2 traindata_v=traindata_vector1 trees_pixel_ndvi=trees_pixel_ndvi2 --overwrite +r.trees.mltrain red_raster=dop_red green_raster=dop_green blue_raster=dop_blue nir_raster=dop_nir ndvi_raster=ndvi_raster ndsm=ndsm slope=slope trees_pixel_ndvi=trees_pixel_ndvi1 trees_raw_v=traindata_vector1 num_samples=4000 group=ml_input_test save_model=test_ml_trees_randomforest.gz --o +r.trees.mlapply area=aoi group=ml_input_test model=test_ml_trees_randomforest.gz output=tree_pixels5 --overwrite +r.trees.postprocess tree_pixels=tree_pixels4 green_raster=dop_green blue_raster=dop_blue nir_raster=dop_nir ndvi_raster=ndvi_raster ndsm=ndsm slope=slope2 nearest=nearest_tree2 peaks=tree_peaks2 trees_raster=tree_objects5 trees_vector=tree_objects5 --overwrite + +* NICHT möglich: Vektor direkt in r.trees.postprocess +Es fehlt eine kleine Abfrage zur Rasterisierung. Wäre simpel hinzuzufügen, falls notwendig. + From fdeb7bdd9b09171f481d7134404f4432225b113b Mon Sep 17 00:00:00 2001 From: vbrunn Date: Wed, 23 Oct 2024 12:13:36 +0200 Subject: [PATCH 04/11] linting, rm Doku_hints --- Doku_hints.md | 23 ------------------- .../r.trees.mltrain/r.trees.mltrain.py | 2 ++ .../r.trees.traindata/r.trees.traindata.py | 11 ++++----- 3 files changed, 7 insertions(+), 29 deletions(-) delete mode 100644 Doku_hints.md diff --git a/Doku_hints.md b/Doku_hints.md deleted file mode 100644 index b603f00..0000000 --- a/Doku_hints.md +++ /dev/null @@ -1,23 +0,0 @@ -## Dokumentations Gedankenstützen - -Änderungen nach r.trees.peaks -* Möglichkeit 1: Raster und direkt in r.trees.postprocess: -r.trees.traindata green_raster=dop_green blue_raster=dop_blue nir_raster=dop_nir ndvi_raster=ndvi_raster ndsm=ndsm slope=slope2 nearest=nearest_tree2 peaks=tree_peaks2 traindata_r=traindata_raster1 trees_pixel_ndvi=trees_pixel_ndvi1 --overwrite -r.trees.postprocess tree_pixels=traindata_raster1 green_raster=dop_green blue_raster=dop_blue nir_raster=dop_nir ndvi_raster=ndvi_raster ndsm=ndsm slope=slope2 nearest=nearest_tree2 peaks=tree_peaks2 trees_raster=tree_objects3 trees_vector=tree_objects3 --overwrite - -* Möglichkeit 2: Raster und ML (r.trees.mltrain, r.trees.mlapply, r.trees.postprocess) -r.trees.traindata green_raster=dop_green blue_raster=dop_blue nir_raster=dop_nir ndvi_raster=ndvi_raster ndsm=ndsm slope=slope2 nearest=> -r.trees.mltrain red_raster=dop_red green_raster=dop_green blue_raster=dop_blue nir_raster=dop_nir ndvi_raster=ndvi_raster ndsm=ndsm slope=slope trees_pixel_ndvi=trees_pixel_ndvi1 trees_raw_r=traindata_raster1 num_samples=8000 group=ml_input_test save_model=test_ml_trees_randomforest.gz --o -r.trees.mlapply area=aoi group=ml_input_test model=test_ml_trees_randomforest.gz output=tree_pixels4 --overwrite -r.trees.postprocess tree_pixels=tree_pixels4 green_raster=dop_green blue_raster=dop_blue nir_raster=dop_nir ndvi_raster=ndvi_raster ndsm=ndsm slope=slope2 nearest=nearest_tree2 peaks=tree_peaks2 trees_raster=tree_objects4 trees_vector=tree_objects4 --overwrite - - -* Möglichkeit 3: Vektor und ML (r.trees.mltrain, r.trees.mlapply, r.trees.postprocess) -r.trees.traindata green_raster=dop_green blue_raster=dop_blue nir_raster=dop_nir ndvi_raster=ndvi_raster ndsm=ndsm slope=slope2 nearest=nearest_tree2 peaks=tree_peaks2 traindata_v=traindata_vector1 trees_pixel_ndvi=trees_pixel_ndvi2 --overwrite -r.trees.mltrain red_raster=dop_red green_raster=dop_green blue_raster=dop_blue nir_raster=dop_nir ndvi_raster=ndvi_raster ndsm=ndsm slope=slope trees_pixel_ndvi=trees_pixel_ndvi1 trees_raw_v=traindata_vector1 num_samples=4000 group=ml_input_test save_model=test_ml_trees_randomforest.gz --o -r.trees.mlapply area=aoi group=ml_input_test model=test_ml_trees_randomforest.gz output=tree_pixels5 --overwrite -r.trees.postprocess tree_pixels=tree_pixels4 green_raster=dop_green blue_raster=dop_blue nir_raster=dop_nir ndvi_raster=ndvi_raster ndsm=ndsm slope=slope2 nearest=nearest_tree2 peaks=tree_peaks2 trees_raster=tree_objects5 trees_vector=tree_objects5 --overwrite - -* NICHT möglich: Vektor direkt in r.trees.postprocess -Es fehlt eine kleine Abfrage zur Rasterisierung. Wäre simpel hinzuzufügen, falls notwendig. - diff --git a/grass-gis-addons/m.analyse.trees/r.trees.mltrain/r.trees.mltrain.py b/grass-gis-addons/m.analyse.trees/r.trees.mltrain/r.trees.mltrain.py index 3345611..9e37b7a 100644 --- a/grass-gis-addons/m.analyse.trees/r.trees.mltrain/r.trees.mltrain.py +++ b/grass-gis-addons/m.analyse.trees/r.trees.mltrain/r.trees.mltrain.py @@ -165,6 +165,7 @@ def cleanup(): grass.run_command("g.remove", type="raster", name=rmrast, **kwargs) grass.del_temp_region() + def numsamplecheck(number_samples, rastername): raster_samples = int(grass.parse_command("r.univar", map=rastername, flags="g")["n"]) if number_samples > raster_samples: @@ -176,6 +177,7 @@ def numsamplecheck(number_samples, rastername): number_samples = raster_samples return number_samples + def main(): global rm_rasters, tmp_mask_old diff --git a/grass-gis-addons/m.analyse.trees/r.trees.traindata/r.trees.traindata.py b/grass-gis-addons/m.analyse.trees/r.trees.traindata/r.trees.traindata.py index f208166..d980aa4 100644 --- a/grass-gis-addons/m.analyse.trees/r.trees.traindata/r.trees.traindata.py +++ b/grass-gis-addons/m.analyse.trees/r.trees.traindata/r.trees.traindata.py @@ -4,10 +4,10 @@ # # MODULE: r.trees.traindata # -# AUTHOR(S): Markus Metz, Lina Krisztian, Guido Riembauer, +# AUTHOR(S): Markus Metz, Lina Krisztian, Guido Riembauer, # Victoria-Leandra Brunn # -# PURPOSE: Creates a preliminary tree map for random forest classification +# PURPOSE: Creates a preliminary tree map for random forest classification # and/or post processing. # # COPYRIGHT: (C) 2023 - 2024 by mundialis and the GRASS Development Team @@ -317,13 +317,12 @@ def main(): if options["trees_pixel_ndvi"]: trees_pixel_ndvi = options["trees_pixel_ndvi"] else: - trees_pixel_ndvi = "trees_pixel_ndvi" - rm_rasters.append(trees_pixel_ndvi) + trees_pixel_ndvi = "trees_pixel_ndvi" + rm_rasters.append(trees_pixel_ndvi) grass.mapcalc( f"{trees_pixel_ndvi} = if({ndvi_split}_max2 < {ndvi_threshold}, null(), {nearest})" ) - # cut to nir: all pixels below 100 are not vegetation # removes shadows with high ndvi e.g. on roofs @@ -503,7 +502,7 @@ def main(): grass.mapcalc(f"{options['traindata_r']} = if(isnull(trees_object_filt_large), null(), 2)") if not options["traindata_v"]: rm_vectors.append("trees_object_filt_large") - + if options["traindata_v"]: grass.run_command( "g.rename", From 937b5a958ddac6287b0b00f81c4dadf25d3ae665 Mon Sep 17 00:00:00 2001 From: vbrunn Date: Wed, 23 Oct 2024 13:30:02 +0200 Subject: [PATCH 05/11] check black --- .../r.trees.mltrain/r.trees.mltrain.py | 18 +++++++++++++----- .../r.trees.traindata/r.trees.traindata.py | 6 ++++-- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/grass-gis-addons/m.analyse.trees/r.trees.mltrain/r.trees.mltrain.py b/grass-gis-addons/m.analyse.trees/r.trees.mltrain/r.trees.mltrain.py index 9e37b7a..aedcc20 100644 --- a/grass-gis-addons/m.analyse.trees/r.trees.mltrain/r.trees.mltrain.py +++ b/grass-gis-addons/m.analyse.trees/r.trees.mltrain/r.trees.mltrain.py @@ -167,12 +167,16 @@ def cleanup(): def numsamplecheck(number_samples, rastername): - raster_samples = int(grass.parse_command("r.univar", map=rastername, flags="g")["n"]) + raster_samples = int( + grass.parse_command("r.univar", map=rastername, flags="g")["n"] + ) if number_samples > raster_samples: grass.warning( - _(f"The chosen number of pixels {number_samples} is exceeding the total number of ", - f"non-null pixels in the given rastermap {rastername}. ", - f"The number of pixels will be set to the maximal amount of {raster_samples}.") + _( + f"The chosen number of pixels {number_samples} is exceeding the total number of ", + f"non-null pixels in the given rastermap {rastername}. ", + f"The number of pixels will be set to the maximal amount of {raster_samples}." + ) ) number_samples = raster_samples return number_samples @@ -295,7 +299,11 @@ def main(): rm_rasters.append("false_trees_trainpnts") rm_rasters.append("notrees_trainpnts") - patch_list = ["trees_trainpnts", "false_trees_trainpnts", "notrees_trainpnts"] + patch_list = [ + "trees_trainpnts", + "false_trees_trainpnts", + "notrees_trainpnts" + ] else: patch_list = [trees_basemap, "false_trees", "notrees"] diff --git a/grass-gis-addons/m.analyse.trees/r.trees.traindata/r.trees.traindata.py b/grass-gis-addons/m.analyse.trees/r.trees.traindata/r.trees.traindata.py index d980aa4..1d0cad7 100644 --- a/grass-gis-addons/m.analyse.trees/r.trees.traindata/r.trees.traindata.py +++ b/grass-gis-addons/m.analyse.trees/r.trees.traindata/r.trees.traindata.py @@ -499,14 +499,16 @@ def main(): # trees: trees_object_filt_large if options["traindata_r"]: - grass.mapcalc(f"{options['traindata_r']} = if(isnull(trees_object_filt_large), null(), 2)") + grass.mapcalc( + f"{options['traindata_r']} = if(isnull(trees_object_filt_large), null(), 2)" + ) if not options["traindata_v"]: rm_vectors.append("trees_object_filt_large") if options["traindata_v"]: grass.run_command( "g.rename", - vector=f"trees_object_filt_large,{options['traindata_v']}" + vector=f"trees_object_filt_large,{options['traindata_v']}", ) From 2df3a5e0c77576f3b0594a8f71762595ba001fea Mon Sep 17 00:00:00 2001 From: vbrunn Date: Wed, 23 Oct 2024 14:30:15 +0200 Subject: [PATCH 06/11] change default dist tree-tree/ tree-building to 50m --- .../m.analyse.trees/r.trees.mltrain/r.trees.mltrain.py | 4 ++-- .../m.analyse.trees/v.trees.param/v.trees.param.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/grass-gis-addons/m.analyse.trees/r.trees.mltrain/r.trees.mltrain.py b/grass-gis-addons/m.analyse.trees/r.trees.mltrain/r.trees.mltrain.py index aedcc20..4568cb0 100644 --- a/grass-gis-addons/m.analyse.trees/r.trees.mltrain/r.trees.mltrain.py +++ b/grass-gis-addons/m.analyse.trees/r.trees.mltrain/r.trees.mltrain.py @@ -175,7 +175,7 @@ def numsamplecheck(number_samples, rastername): _( f"The chosen number of pixels {number_samples} is exceeding the total number of ", f"non-null pixels in the given rastermap {rastername}. ", - f"The number of pixels will be set to the maximal amount of {raster_samples}." + f"The number of pixels will be set to the maximal amount of {raster_samples}.", ) ) number_samples = raster_samples @@ -302,7 +302,7 @@ def main(): patch_list = [ "trees_trainpnts", "false_trees_trainpnts", - "notrees_trainpnts" + "notrees_trainpnts", ] else: patch_list = [trees_basemap, "false_trees", "notrees"] diff --git a/grass-gis-addons/m.analyse.trees/v.trees.param/v.trees.param.py b/grass-gis-addons/m.analyse.trees/v.trees.param/v.trees.param.py index 0f0802e..c77c09f 100755 --- a/grass-gis-addons/m.analyse.trees/v.trees.param/v.trees.param.py +++ b/grass-gis-addons/m.analyse.trees/v.trees.param/v.trees.param.py @@ -71,7 +71,7 @@ # % type: integer # % required: yes # % label: Range in which is searched for neighbouring buildings -# % answer: 500 +# % answer: 50 # % guisection: Parameters # %end @@ -80,7 +80,7 @@ # % type: integer # % required: yes # % label: Range in which is searched for neighbouring trees -# % answer: 500 +# % answer: 50 # % guisection: Parameters # %end From b052d91e7b39ad997d541cc6159edc7136c3be64 Mon Sep 17 00:00:00 2001 From: vbrunn Date: Wed, 23 Oct 2024 14:52:07 +0200 Subject: [PATCH 07/11] update top-level README --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 832bf82..db9a39a 100644 --- a/README.md +++ b/README.md @@ -39,8 +39,9 @@ Repo for code and script transfer between mundialis and RVR - GRASS GIS addons: - **m.analyse.trees** - **r.trees.peaks** assigns pixels to nearest peak (tree crown). - - **r.trees.mltrain** generates training data for a machine learning (ML) - model to detect trees and trains the model with these training data. + - **r.trees.peaks** generates training data for a machine learning (ML) + model to detect trees and provides a preliminray tree candidate map in either vector or raster format. + - **r.trees.mltrain** trains the ML model with the training data from before or own training data. - **r.trees.mlapply** applies the tree classification model in parallel to the area of interest (current region). - **r.trees.mlapply.worker** applies classification model for a region From b89af01897939d96ecff6da2bba942a1c8016fce Mon Sep 17 00:00:00 2001 From: vbrunn Date: Wed, 23 Oct 2024 16:01:58 +0200 Subject: [PATCH 08/11] change column name of centroid trunk position --- .../v.trees.param.worker/v.trees.param.worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grass-gis-addons/m.analyse.trees/v.trees.param.worker/v.trees.param.worker.py b/grass-gis-addons/m.analyse.trees/v.trees.param.worker/v.trees.param.worker.py index 76f2a49..7423652 100644 --- a/grass-gis-addons/m.analyse.trees/v.trees.param.worker/v.trees.param.worker.py +++ b/grass-gis-addons/m.analyse.trees/v.trees.param.worker/v.trees.param.worker.py @@ -341,7 +341,7 @@ def treetrunk(list_attr, treecrowns): # can be used as an estimate of the trunk position. grass.message(_("Calculating tree trunk position...")) # Centroid as tree trunk position - col_sp_cent = "pos_cent" + col_sp_cent = "pos_rand" # change appropriately after r.to.vect is fixed if f"{col_sp_cent}_x" in list_attr: grass.warning( _( From 8fef389d0ea2ccb7fb6ecf23b2e35e795438e100 Mon Sep 17 00:00:00 2001 From: vbrunn Date: Wed, 23 Oct 2024 16:07:04 +0200 Subject: [PATCH 09/11] linting --- .../v.trees.param.worker/v.trees.param.worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grass-gis-addons/m.analyse.trees/v.trees.param.worker/v.trees.param.worker.py b/grass-gis-addons/m.analyse.trees/v.trees.param.worker/v.trees.param.worker.py index 7423652..d3360a5 100644 --- a/grass-gis-addons/m.analyse.trees/v.trees.param.worker/v.trees.param.worker.py +++ b/grass-gis-addons/m.analyse.trees/v.trees.param.worker/v.trees.param.worker.py @@ -341,7 +341,7 @@ def treetrunk(list_attr, treecrowns): # can be used as an estimate of the trunk position. grass.message(_("Calculating tree trunk position...")) # Centroid as tree trunk position - col_sp_cent = "pos_rand" # change appropriately after r.to.vect is fixed + col_sp_cent = "pos_rand" # change appropriately after r.to.vect is fixed if f"{col_sp_cent}_x" in list_attr: grass.warning( _( From 50c48a478bc909860f910196c97916f6ba8657b7 Mon Sep 17 00:00:00 2001 From: vbrunn Date: Wed, 23 Oct 2024 16:13:13 +0200 Subject: [PATCH 10/11] README suggested changes --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index db9a39a..2f50c8d 100644 --- a/README.md +++ b/README.md @@ -39,8 +39,8 @@ Repo for code and script transfer between mundialis and RVR - GRASS GIS addons: - **m.analyse.trees** - **r.trees.peaks** assigns pixels to nearest peak (tree crown). - - **r.trees.peaks** generates training data for a machine learning (ML) - model to detect trees and provides a preliminray tree candidate map in either vector or raster format. + - **r.trees.traindata** generates training data for a machine learning (ML) model + to detect trees and provides a preliminray tree candidate map in either vector or raster format. - **r.trees.mltrain** trains the ML model with the training data from before or own training data. - **r.trees.mlapply** applies the tree classification model in parallel to the area of interest (current region). From 15756602a5de6bf10784efea3954d749c553f1e3 Mon Sep 17 00:00:00 2001 From: vbrunn Date: Thu, 24 Oct 2024 09:37:03 +0200 Subject: [PATCH 11/11] add suggested changes --- .../r.trees.traindata/r.trees.traindata.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/grass-gis-addons/m.analyse.trees/r.trees.traindata/r.trees.traindata.py b/grass-gis-addons/m.analyse.trees/r.trees.traindata/r.trees.traindata.py index 1d0cad7..3afcd2e 100644 --- a/grass-gis-addons/m.analyse.trees/r.trees.traindata/r.trees.traindata.py +++ b/grass-gis-addons/m.analyse.trees/r.trees.traindata/r.trees.traindata.py @@ -143,16 +143,16 @@ # %option G_OPT_R_OUTPUT # % key: traindata_r -# % label: primary tree map as raster -# % description: Primary tree map in raster format to pass to mltrain +# % label: Preliminary tree map as raster +# % description: Preliminary tree map in raster format to pass to mltrain # % required: no # % guisection: Output # %end # %option G_OPT_V_OUTPUT # % key: traindata_v -# % label: primary tree map as vector -# % description: Primary tree map in vector format to export and edit +# % label: Preliminary tree map as vector +# % description: Preliminary tree map in vector format to export and edit # % required: no # % guisection: Output # %end