From d2efc5d5f9a5eeb672d2f011c3236a273fffd9f1 Mon Sep 17 00:00:00 2001 From: jhalbauer Date: Thu, 13 Jun 2024 11:59:57 +0200 Subject: [PATCH 1/3] r.dop.import: added SN worker --- r.dop.import.sn/r.dop.import.sn.html | 10 + r.dop.import.sn/r.dop.import.sn.py | 275 ++++++++------ r.dop.import.worker.sn/Makefile | 8 + .../r.dop.import.worker.sn.html | 18 + .../r.dop.import.worker.sn.py | 347 ++++++++++++++++++ testsuite/test_r_dop_import_SN.py | 2 +- 6 files changed, 548 insertions(+), 112 deletions(-) create mode 100644 r.dop.import.worker.sn/Makefile create mode 100644 r.dop.import.worker.sn/r.dop.import.worker.sn.html create mode 100644 r.dop.import.worker.sn/r.dop.import.worker.sn.py diff --git a/r.dop.import.sn/r.dop.import.sn.html b/r.dop.import.sn/r.dop.import.sn.html index 3c90437..13258ae 100644 --- a/r.dop.import.sn/r.dop.import.sn.html +++ b/r.dop.import.sn/r.dop.import.sn.html @@ -3,6 +3,16 @@

DESCRIPTION

r.dop.import.sn downloads and imports digital orthophotos (DOP) for Sachsen (SN) and area of interest. +The data "Digitale Orthophotos" is downloaded from +Geoportal Sachsen +and can be used under the specification of the license and referencing the source: + +
source: GeoSN + +
license: Datenlizenz Deutschland Namensnennung 2.0, + +
DOP +

EXAMPLES

diff --git a/r.dop.import.sn/r.dop.import.sn.py b/r.dop.import.sn/r.dop.import.sn.py index 1fbb9c1..92558de 100644 --- a/r.dop.import.sn/r.dop.import.sn.py +++ b/r.dop.import.sn/r.dop.import.sn.py @@ -5,7 +5,7 @@ # MODULE: r.dop.import.sn # AUTHOR(S): Johannes Halbauer, Anika Weinmann # -# PURPOSE: Downloads DOPs for Sachsen and AOI +# PURPOSE: Downloads DOPs for Sachsen and aoi # COPYRIGHT: (C) 2024 by mundialis GmbH & Co. KG and the GRASS # Development Team # @@ -22,7 +22,7 @@ ############################################################################ # %module -# % description: Downloads DOPs for Sachsen and AOI. +# % description: Downloads DOPs for Sachsen and aoi. # % keyword: raster # % keyword: import # % keyword: DOP @@ -37,8 +37,8 @@ # %option # % key: download_dir -# % label: Path of output folder -# % description: Path of download folder +# % label: Path to output folder +# % description: Path to download folder # % required: no # % multiple: no # %end @@ -47,6 +47,16 @@ # % description: Name for output raster map # %end +# %option +# % key: nprocs +# % type: integer +# % required: no +# % multiple: no +# % label: Number of parallel processes +# % description: Number of cores for multiprocessing, -2 is the number of available cores - 1 +# % answer: -2 +# %end + # %option G_OPT_MEMORYMB # %end @@ -66,43 +76,49 @@ import atexit import os -from osgeo import gdal -from time import sleep +import sys + import grass.script as grass +from grass.pygrass.modules import Module, ParallelModuleQueue +from grass.pygrass.utils import get_lib_path +from osgeo import gdal from grass_gis_helpers.cleanup import general_cleanup -from grass_gis_helpers.general import test_memory -from grass_gis_helpers.open_geodata_germany.download_data import ( - check_download_dir, - download_data_using_threadpool, - extract_compressed_files, -) -from grass_gis_helpers.raster import adjust_raster_resolution, create_vrt from grass_gis_helpers.data_import import ( download_and_import_tindex, get_list_of_tindex_locations, ) - -# set global varibales +from grass_gis_helpers.general import test_memory +from grass_gis_helpers.open_geodata_germany.download_data import ( + check_download_dir, +) +from grass_gis_helpers.raster import create_vrt + +# import module library +path = get_lib_path(modname="r.dop.import") +if path is None: + grass.fatal("Unable to find the dop library directory.") +sys.path.append(path) +try: + from r_dop_import_lib import setup_parallel_processing +except Exception as imp_err: + grass.fatal(f"r.dop.import library could not be imported: {imp_err}") + +# set global variables TINDEX = ( "https://github.com/mundialis/tile-indices/raw/main/DOP/SN/" "DOP20_tileindex_SN.gpkg.gz" ) ID = grass.tempname(12) -RETRIES = 10 orig_region = None -keep_data = False rm_rasters = [] rm_vectors = [] download_dir = None +rm_dirs = [] def cleanup(): - rm_dirs = [] - if not keep_data: - if download_dir: - rm_dirs.append(download_dir) general_cleanup( orig_region=orig_region, rm_rasters=rm_rasters, @@ -112,121 +128,158 @@ def cleanup(): def main(): - global ID, orig_region, rm_rasters, rm_vectors, keep_data, download_dir + global ID, orig_region, rm_rasters, rm_vectors + global temp_loc_path, download_dir, new_mapset aoi = options["aoi"] download_dir = check_download_dir(options["download_dir"]) + nprocs = int(options["nprocs"]) + nprocs = setup_parallel_processing(nprocs) output = options["output"] - keep_data = flags["k"] - native_res = flags["r"] + fs = "SN" # set memory to input if possible options["memory"] = test_memory(options["memory"]) + # create list for each raster band for building entire raster + all_raster = { + "red": [], + "green": [], + "blue": [], + "nir": [], + } + # save original region orig_region = f"original_region_{ID}" grass.run_command("g.region", save=orig_region, quiet=True) - ns_res = grass.region()["nsres"] + + # get region resolution and check if resolution consistent + reg = grass.region() + if reg["nsres"] == reg["ewres"]: + ns_res = reg["nsres"] + else: + grass.fatal("N/S resolution is not the same as E/W resolution!") # set region if aoi is given if aoi: grass.run_command("g.region", vector=aoi, flags="a") + # if no aoi save region as aoi + else: + aoi = f"region_aoi_{ID}" + grass.run_command( + "v.in.region", + output=aoi, + quiet=True, + ) # get tile index tindex_vect = f"dop_tindex_{ID}" rm_vectors.append(tindex_vect) download_and_import_tindex(TINDEX, tindex_vect, download_dir) - # get download urls which overlap with AOI - # or current region if no AOI is given + # get download urls which overlap with aoi + # or current region if no aoi is given url_tiles = get_list_of_tindex_locations(tindex_vect, aoi) - # if k flag is set download DOPs to download_dir - if keep_data: - download_list = [ - os.path.dirname(url.replace("/vsizip/vsicurl/", "")) - for url in url_tiles - ] - - # download with using nprocs=3 - download_data_using_threadpool(download_list, download_dir, 3) - # extract downloaded files - file_names = [os.path.basename(url) for url in download_list] - extract_compressed_files(file_names, download_dir) - - # import DOPs directly - grass.message(_("Importing DOPs...")) - - # create list of lists for all DOPs caused by GRASS import structure - # (one raster map per band) - all_dops = [[], [], [], []] - res = None - for url in url_tiles: - # define name for imported DOP - dop_name = os.path.splitext(os.path.basename(url))[0].replace("-", "") - - # define input parameter for import - if keep_data: - input = os.path.join(download_dir, os.path.basename(url)) - else: - if "/vsicurl/" not in url: - input = f"/vsicurl/{url}" + url_tiles_count = 0 + for i in range(len(url_tiles)): + url_tiles_count += 1 + url_tiles[i] = (url_tiles_count, [url_tiles[i]]) + number_tiles = len(url_tiles) + + # set number of parallel processes to number of tiles + if number_tiles < nprocs: + nprocs = number_tiles + queue = ParallelModuleQueue(nprocs=nprocs) + + # get GISDBASE and Location + gisenv = grass.gisenv() + gisdbase = gisenv["GISDBASE"] + location = gisenv["LOCATION_NAME"] + + # set queue and variables for worker adddon + try: + grass.message( + _(f"Importing {number_tiles} DOPs for SN in parallel...") + ) + for tile in url_tiles: + key = tile[0] + new_mapset = f"tmp_mapset_rdop_import_tile_{key}_{os.getpid()}" + rm_dirs.append(os.path.join(gisdbase, location, new_mapset)) + b_name = os.path.basename(tile[1][0]) + raster_name = ( + f"{b_name.split('.')[0].replace('-', '_')}" f"_{os.getpid()}" + ) + for key_rast in all_raster: + all_raster[key_rast].append( + f"{fs}_{raster_name}_{key_rast}@{new_mapset}" + ) + param = { + "tile_key": key, + "tile_url": tile[1][0], + "raster_name": raster_name, + "orig_region": orig_region, + "memory": 1000, + "new_mapset": new_mapset, + "flags": "", + } + grass.message(_(f"raster name: {raster_name}")) + + # modify params + if aoi: + param["aoi"] = aoi + if options["download_dir"]: + param["download_dir"] = download_dir + if flags["k"]: + param["flags"] += "k" + if flags["r"]: + dop_src = gdal.Open(param["tile_url"]) + param["resolution_to_import"] = abs( + dop_src.GetGeoTransform()[1] + ) else: - input = url - - if res is None: - ds = gdal.Open(url) - count = 0 - while ds is None and count <= RETRIES: - count += 1 - sleep(10) - ds = gdal.Open(url) - res = ds.GetGeoTransform()[1] - - # import DOPs - count = 0 - imported = False - while not imported and count <= RETRIES: - count += 1 - try: - grass.run_command( - "r.import", - input=input, - output=dop_name, - extent="region", - resolution="value", - resolution_value=res, - memory=1000, - overwrite=True, - quiet=True, + param["resolution_to_import"] = ns_res + + # append raster bands to download to remove list + rm_red = f"{raster_name}_red" + rm_green = f"{raster_name}_green" + rm_blue = f"{raster_name}_blue" + rm_nir = f"{raster_name}_nir" + rm_rasters.append(rm_red) + rm_rasters.append(rm_green) + rm_rasters.append(rm_blue) + rm_rasters.append(rm_nir) + + # run worker addon in parallel + r_dop_import_worker_SN = Module( + "r.dop.import.worker.sn", + **param, + run_=False, + ) + # catch all GRASS output to stdout and stderr + r_dop_import_worker_SN.stdout = grass.PIPE + r_dop_import_worker_SN.stderr = grass.PIPE + queue.put(r_dop_import_worker_SN) + queue.wait() + except Exception: + for proc_num in range(queue.get_num_run_procs()): + proc = queue.get(proc_num) + if proc.returncode != 0: + # save all stderr to a variable and pass it to a GRASS + # exception + errmsg = proc.outputs["stderr"].value.strip() + grass.fatal( + _(f"\nERROR by processing <{proc.get_bash()}>: {errmsg}") ) - imported = True - except Exception: - sleep(10) - # append DOP name with band suffix - all_dops[0].append(dop_name + ".1") - all_dops[1].append(dop_name + ".2") - all_dops[2].append(dop_name + ".3") - all_dops[3].append(dop_name + ".4") - - # create VRT - vrt_outputs = [] - for dops, band in zip(all_dops, ["red", "green", "blue", "nir"]): - vrt_output = f"{output}_{band}" - create_vrt(dops, vrt_output) - vrt_outputs.append(vrt_output) - - # resample / interpolate whole VRT (because interpolating single files leads - # to empty rows and columns) - # check resolution and resample / interpolate data if needed - if not native_res: - grass.message(_("Resampling / interpolating data...")) - grass.run_command("g.region", raster=vrt_output, res=ns_res, flags="a") - for vrt_out in vrt_outputs: - adjust_raster_resolution(vrt_out, vrt_out, ns_res) - - for result in vrt_outputs: - grass.message(_(f"DOP raster map <{result}> is created.")) + + # create one vrt per band of all imported DOPs + raster_out = [] + for band, b_list in all_raster.items(): + out = f"{output}_{band}" + create_vrt(b_list, out) + raster_out.append(out) + + grass.message(_(f"Generated following raster maps: {raster_out}")) if __name__ == "__main__": diff --git a/r.dop.import.worker.sn/Makefile b/r.dop.import.worker.sn/Makefile new file mode 100644 index 0000000..be9559b --- /dev/null +++ b/r.dop.import.worker.sn/Makefile @@ -0,0 +1,8 @@ +MODULE_TOPDIR = ../.. + +PGM = r.dop.import.worker.sn + +include $(MODULE_TOPDIR)/include/Make/Python.make +include $(MODULE_TOPDIR)/include/Make/Script.make + +default: script diff --git a/r.dop.import.worker.sn/r.dop.import.worker.sn.html b/r.dop.import.worker.sn/r.dop.import.worker.sn.html new file mode 100644 index 0000000..0679a78 --- /dev/null +++ b/r.dop.import.worker.sn/r.dop.import.worker.sn.html @@ -0,0 +1,18 @@ +

DESCRIPTION

+ +r.dop.import.worker.sn is used within r.dop.import.sn to import the +DOPs in parallel. + +

SEE ALSO

+ + +r.dop.import.bb.be, +r.buildvrt, +r.import, +v.check.federal_state + + +

AUTHORS

+ +Johannes Halbauer, mundialis GmbH & Co. KG +Lina Krisztian, mundialis GmbH & Co. KG \ No newline at end of file diff --git a/r.dop.import.worker.sn/r.dop.import.worker.sn.py b/r.dop.import.worker.sn/r.dop.import.worker.sn.py new file mode 100644 index 0000000..8e4d2b9 --- /dev/null +++ b/r.dop.import.worker.sn/r.dop.import.worker.sn.py @@ -0,0 +1,347 @@ +#!/usr/bin/env python3 +# +############################################################################ +# +# MODULE: r.dop.import.worker.sn +# AUTHOR(S): Johannes Halbauer & Lina Krisztian +# +# PURPOSE: Downloads Digital Orthophotos (DOPs) within a specified area in Sachsen +# COPYRIGHT: (C) 2024 by mundialis GmbH & Co. KG and the GRASS Development Team +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +############################################################################# + +# %Module +# % description: Downloads and imports single Digital Orthophotos (DOPs) in Sachsen +# % keyword: imagery +# % keyword: download +# % keyword: DOP +# %end + +# %option G_OPT_V_INPUT +# % key: aoi +# % required: no +# % description: Vector map to restrict DOP import to +# %end + +# %option +# % key: download_dir +# % label: Path to output folder +# % description: Path to download folder +# % required: no +# % multiple: no +# %end + +# %option +# % key: tile_key +# % required: yes +# % description: Key of tile-DOP to import +# %end + +# %option +# % key: tile_url +# % required: yes +# % description: URL of tile-DOP to import +# %end + +# %option +# % key: new_mapset +# % type: string +# % required: yes +# % multiple: no +# % key_desc: name +# % description: Name for new mapset +# %end + +# %option +# % key: orig_region +# % required: yes +# % description: Original region +# %end + +# %option +# % key: resolution_to_import +# % required: yes +# % description: Resolution of region, for which DOP will be imported +# %end + +# %option G_OPT_R_OUTPUT +# % key: raster_name +# % description: Name of raster output +# %end + +# %option G_OPT_MEMORYMB +# % description: Memory which is used by all processes (it is divided by nprocs for each single parallel process) +# %end + +# %flag +# % key: r +# % description: Use native DOP resolution +# %end + +# %flag +# % key: k +# % label: Keep downloaded data in the download directory +# %end + +# %rules +# % requires_all: -k,download_dir +# %end + + +import atexit +import os +import sys +from time import sleep + +import grass.script as grass +from grass.pygrass.utils import get_lib_path + +from grass_gis_helpers.cleanup import general_cleanup, cleaning_tmp_location +from grass_gis_helpers.general import test_memory +from grass_gis_helpers.location import ( + get_current_location, + create_tmp_location, + switch_back_original_location, +) +from grass_gis_helpers.mapset import switch_to_new_mapset +from grass_gis_helpers.open_geodata_germany.download_data import ( + download_data_using_threadpool, + extract_compressed_files, +) +from grass_gis_helpers.raster import adjust_raster_resolution + +# import module library +path = get_lib_path(modname="r.dop.import") +if path is None: + grass.fatal("Unable to find the dop library directory.") +sys.path.append(path) +try: + from r_dop_import_lib import rescale_to_1_256 +except Exception as imp_err: + grass.fatal(f"r.dop.import library could not be imported: {imp_err}") + +rm_rast = [] +rm_group = [] + +gisdbase = None +tmp_loc = None +tmp_gisrc = None + +RETRIES = 30 +WAITING_TIME = 10 + + +def cleanup(): + cleaning_tmp_location( + None, tmp_loc=tmp_loc, tmp_gisrc=tmp_gisrc, gisdbase=gisdbase + ) + general_cleanup( + rm_rasters=rm_rast, + rm_groups=rm_group, + ) + + +def import_and_reproject( + url, + raster_name, + resolution_to_import, + aoi_map=None, + download_dir=None, + epsg=None, +): + """Import DOPs and reproject them if needed. + + Args: + url (str): The URL of the DOP to import + raster_name (str): The prefix name for the output rasters + aoi_map (str): Name of AOI vector map + download_dir (str): Path to local directory to downlaod DOPs to + epsg (int): EPSG code which has to be set if the reproduction should be + done manually and not by r.import + """ + global gisdbase, tmp_loc, tmp_gisrc + aoi_map_to_set_region1 = aoi_map + + # get actual location, mapset, ... + loc, mapset, gisdbase, gisrc = get_current_location() + if not aoi_map: + aoi_map = f"region_aoi_{grass.tempname(8)}" + aoi_map_to_set_region1 = aoi_map + grass.run_command("v.in.region", output=aoi_map, quiet=True) + else: + aoi_map_mapset = aoi_map.split("@") + aoi_map_to_set_region1 = aoi_map_mapset[0] + if len(aoi_map_mapset) == 2: + mapset = aoi_map_mapset[1] + + # create temporary location with EPSG:25832 + tmp_loc, tmp_gisrc = create_tmp_location(epsg) + + # reproject aoi + if aoi_map: + grass.run_command( + "v.proj", + location=loc, + mapset=mapset, + input=aoi_map_to_set_region1, + output=aoi_map_to_set_region1, + quiet=True, + ) + grass.run_command( + "g.region", + vector=aoi_map_to_set_region1, + res=resolution_to_import, + flags="a", + ) + + # import data + # set memory manually to 1000 + # Process stuck, when memory is too large (100000) + # GDAL_CACHEMAX wird nur als MB interpretiert + kwargs = { + "input": url, + "output": raster_name, + "memory": 1000, + "quiet": True, + "extent": "region", + } + # TODO: Auslagern dieser Funktion in Lib, jedoch unterscheidet + # sich der folgende if-Block bspw. von dem von BB/BE + # + # download and keep data to download dir if -k flag ist set + # and change input parameter in kwargs to local path + if flags["k"]: + basename = os.path.basename(url) + print(basename) + url = os.path.dirname(url.replace("/vsizip/vsicurl/", "")) + download_data_using_threadpool([url], download_dir, None) + extract_compressed_files([os.path.basename(url)], download_dir) + kwargs["input"] = os.path.join(download_dir, basename) + + import_sucess = False + tries = 0 + while not import_sucess: + tries += 1 + if tries > RETRIES: + grass.fatal( + _( + f"Importing {kwargs['input']} failed after {RETRIES} " + "retries." + ) + ) + try: + grass.run_command("r.import", **kwargs) + import_sucess = True + except Exception: + sleep(WAITING_TIME) + if not aoi_map: + grass.run_command("g.region", raster=f"{raster_name}.1") + + # reproject data + res = float( + grass.parse_command("r.info", flags="g", map=f"{raster_name}.1")[ + "nsres" + ] + ) + # switch location + os.environ["GISRC"] = str(gisrc) + if aoi_map: + grass.run_command("g.region", vector=aoi_map, res=res, flags="a") + else: + grass.run_command("g.region", res=res, flags="a") + for i in range(1, 5): + name = f"{raster_name}.{i}" + # set memory manually to 1000 + # Process stuck, when memory is too large (100000) + # GDAL_CACHEMAX is only interpreted as MB, if value is <100000 + grass.run_command( + "r.proj", + location=tmp_loc, + mapset="PERMANENT", + input=name, + output=name, + resolution=res, + flags="n", + quiet=True, + memory=1000, + ) + + +def main(): + # parser options + tile_key = options["tile_key"] + tile_url = options["tile_url"] + raster_name = options["raster_name"] + resolution_to_import = float(options["resolution_to_import"]) + orig_region = options["orig_region"] + new_mapset = options["new_mapset"] + download_dir = options["download_dir"] + + # set memory to input if possible + options["memory"] = test_memory(options["memory"]) + + # switch to new mapset for parallel processing + gisrc, newgisrc, old_mapset = switch_to_new_mapset(new_mapset) + + # set region + grass.run_command("g.region", region=f"{orig_region}@{old_mapset}") + if options["aoi"]: + aoi_map = f"{options['aoi']}@{old_mapset}" + else: + aoi_map = None + + # import DOP tile with original resolution + grass.message( + _(f"Started DOP import for key: {tile_key} and URL: {tile_url}") + ) + + # import and reproject DOP tiles based on tileindex + import_and_reproject( + tile_url, + raster_name, + resolution_to_import, + aoi_map, + download_dir, + epsg=25832, + ) + # adjust resolution if required + for band in [1, 2, 3, 4]: + raster_name_band = f"{raster_name}.{band}" + grass.run_command( + "g.region", + raster=raster_name_band, + res=resolution_to_import, + flags="a", + ) + adjust_raster_resolution( + raster_name_band, raster_name_band, resolution_to_import + ) + rm_group.append(raster_name) + grass.message(_(f"Finishing raster import for {raster_name}...")) + + # rescale imported DOPs + new_rm_rast = rescale_to_1_256("SN", raster_name) + rm_rast.extend(new_rm_rast) + + # switch back to original location + switch_back_original_location(gisrc) + grass.utils.try_remove(newgisrc) + grass.message( + _(f"DOP import for key: {tile_key} and URL: {tile_url} done!") + ) + + +if __name__ == "__main__": + options, flags = grass.parser() + atexit.register(cleanup) + main() diff --git a/testsuite/test_r_dop_import_SN.py b/testsuite/test_r_dop_import_SN.py index 031a84b..cf1e668 100644 --- a/testsuite/test_r_dop_import_SN.py +++ b/testsuite/test_r_dop_import_SN.py @@ -28,7 +28,7 @@ class TestRDopImportSN(RDopImportTestBase): fs = "SN" ref_res = 0.2 - aoi_cells = 2021300 + aoi_cells = 2027082 def test_default_settings(self): """ From f5c72b9b5775fb9c0895aad1010900283135ec13 Mon Sep 17 00:00:00 2001 From: jhalbauer Date: Thu, 13 Jun 2024 14:28:01 +0200 Subject: [PATCH 2/3] small naming changes --- r.dop.import.bb.be/r.dop.import.bb.be.html | 2 +- r.dop.import.worker.nw/r.dop.import.worker.nw.html | 4 ++-- r.dop.import.worker.sn/r.dop.import.worker.sn.html | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/r.dop.import.bb.be/r.dop.import.bb.be.html b/r.dop.import.bb.be/r.dop.import.bb.be.html index f5f1ac3..e75ce95 100644 --- a/r.dop.import.bb.be/r.dop.import.bb.be.html +++ b/r.dop.import.bb.be/r.dop.import.bb.be.html @@ -1,6 +1,6 @@

DESCRIPTION

-r.dop.import.bb downloads and imports digital orthophotos (DOP) +r.dop.import.bb.be downloads and imports digital orthophotos (DOP) for Brandenburg and area of interest. The data "Digitale Orthophotos" is downloaded from diff --git a/r.dop.import.worker.nw/r.dop.import.worker.nw.html b/r.dop.import.worker.nw/r.dop.import.worker.nw.html index 205bd1c..9050cdd 100644 --- a/r.dop.import.worker.nw/r.dop.import.worker.nw.html +++ b/r.dop.import.worker.nw/r.dop.import.worker.nw.html @@ -6,7 +6,7 @@

DESCRIPTION

SEE ALSO

-r.dop.import.bb.be, +r.dop.import.nw, r.buildvrt, r.import, v.check.federal_state @@ -15,4 +15,4 @@

SEE ALSO

AUTHORS

Johannes Halbauer, mundialis GmbH & Co. KG -Lina Krisztian, mundialis GmbH & Co. KG \ No newline at end of file +Lina Krisztian, mundialis GmbH & Co. KG diff --git a/r.dop.import.worker.sn/r.dop.import.worker.sn.html b/r.dop.import.worker.sn/r.dop.import.worker.sn.html index 0679a78..5e6148b 100644 --- a/r.dop.import.worker.sn/r.dop.import.worker.sn.html +++ b/r.dop.import.worker.sn/r.dop.import.worker.sn.html @@ -6,7 +6,7 @@

DESCRIPTION

SEE ALSO

-r.dop.import.bb.be, +r.dop.import.sn, r.buildvrt, r.import, v.check.federal_state @@ -15,4 +15,4 @@

SEE ALSO

AUTHORS

Johannes Halbauer, mundialis GmbH & Co. KG -Lina Krisztian, mundialis GmbH & Co. KG \ No newline at end of file +Lina Krisztian, mundialis GmbH & Co. KG From 3e83773b4d311091c5ab553835a98c5457663815 Mon Sep 17 00:00:00 2001 From: jhalbauer Date: Wed, 26 Jun 2024 14:03:32 +0200 Subject: [PATCH 3/3] moved import func in lib, deleted old addon --- lib_dop/r_dop_import_lib.py | 214 ++++- r.dop.import.old/Makefile | 10 - r.dop.import.old/download_urls.py | 29 - r.dop.import.old/federal_state_info.py | 57 -- r.dop.import.old/install_required_addons.sh | 1 - r.dop.import.old/r.dop.import.old.html | 103 --- r.dop.import.old/r.dop.import.old.py | 803 ------------------ .../r.dop.import.worker.bb.be.py | 164 +--- .../r.dop.import.worker.he.py | 6 +- .../r.dop.import.worker.nw.py | 154 +--- .../r.dop.import.worker.sn.py | 154 +--- .../r.dop.import.worker.th.py | 6 +- r.dop.import/r.dop.import.py | 26 +- testsuite/test_r_dop_import_SN.py | 2 +- 14 files changed, 258 insertions(+), 1471 deletions(-) delete mode 100644 r.dop.import.old/Makefile delete mode 100644 r.dop.import.old/download_urls.py delete mode 100644 r.dop.import.old/federal_state_info.py delete mode 100644 r.dop.import.old/install_required_addons.sh delete mode 100644 r.dop.import.old/r.dop.import.old.html delete mode 100644 r.dop.import.old/r.dop.import.old.py diff --git a/lib_dop/r_dop_import_lib.py b/lib_dop/r_dop_import_lib.py index 815888e..dab4cb5 100644 --- a/lib_dop/r_dop_import_lib.py +++ b/lib_dop/r_dop_import_lib.py @@ -26,6 +26,14 @@ import grass.script as grass from grass_gis_helpers.general import set_nprocs +from grass_gis_helpers.location import ( + get_current_location, + create_tmp_location, +) +from grass_gis_helpers.open_geodata_germany.download_data import ( + download_data_using_threadpool, + extract_compressed_files, +) from grass_gis_helpers.raster import adjust_raster_resolution, rename_raster OPEN_DATA_AVAILABILITY = { @@ -34,6 +42,9 @@ "SUPPORTED": ["BE", "BB", "NW", "SN", "TH", "HE"], } +RETRIES = 30 +WAITING_TIME = 10 + def setup_parallel_processing(nprocs): nprocs = set_nprocs(nprocs) @@ -78,7 +89,7 @@ def rescale_to_1_256(prefix, raster_name, extension="num"): def create_grid_and_tiles_list( ns_res, ew_res, tile_size, grid, rm_vectors, aoi, ID, fs ): - """Check if aoi is smaller than grid tile size and creates grid if not. + """Check if aoi is smaller than grid tile size and create grid if not. Also create a list containing tiles which overlap with the aoi. Args: ns_res (float): Vertical resolution @@ -257,3 +268,204 @@ def import_dop_from_wms( grass.run_command( "g.remove", type="raster", pattern=f"{cir_band}_*_tmp*", flags="f" ) + + +def keep_data_NW(url, download_dir): + """Download and keep DOPs for NW from url using threadpool + + Args: + url (string): Url to download data from + download_dir (str): Path to directory to download data to + + Returns: + (str): Path to download data + """ + url = url.replace("/vsicurl/", "") + basename = os.path.basename(url) + download_data_using_threadpool([url], download_dir, None) + + return os.path.join(download_dir, basename) + + +def keep_data_BB_BE(url, download_dir): + """Download and keep DOPs for BB/BE from url using threadpool + + Args: + url (string): Url to download data from + download_dir (str): Path to directory to download data to + + Returns: + (str): Path to download data + """ + url = url.replace("/vsizip/vsicurl/", "") + basename = os.path.basename(url) + url = url.replace(basename, "")[:-1] + download_data_using_threadpool([url], download_dir, None) + extract_compressed_files([basename.replace(".tif", ".zip")], download_dir) + return os.path.join(download_dir, basename) + + +def keep_data_SN(url, download_dir): + """Download and keep DOPs for BB/BE from url using threadpool + + Args: + url (string): Url to download data from + download_dir (str): Path to directory to download data to + + Returns: + (str): Path to download data + """ + basename = os.path.basename(url) + print(basename) + url = os.path.dirname(url.replace("/vsizip/vsicurl/", "")) + download_data_using_threadpool([url], download_dir, None) + extract_compressed_files([os.path.basename(url)], download_dir) + + return os.path.join(download_dir, basename) + + +def import_and_reproject( + url, + raster_name, + resolution_to_import, + fs, + aoi_map=None, + download_dir=None, + epsg=None, + keep_data=None, +): + """Import DOPs and reproject them if needed. + + Args: + url (str): The URL of the DOP to import + raster_name (str): The prefix name for the output rasters + resolution_to_import (float): Resolution to use for region and raster import + fs (str): Abbreviation of the current federal state + aoi_map (str): Name of AOI vector map + download_dir (str): Path to local directory to downlaod DOPs to + epsg (int): EPSG code which has to be set if the reproduction should be + done manually and not by r.import + keep_data (bool): Download raster data to local directory and keep it + + Returns: + gisdbase (str): Path to GISDBASE + tmp_loc (str): Name of temporary location + tmp_gisrc (str): Path to GISRC file + """ + aoi_map_to_set_region1 = aoi_map + + # get actual location, mapset, ... + loc, mapset, gisdbase, gisrc = get_current_location() + if not aoi_map: + aoi_map = f"region_aoi_{grass.tempname(8)}" + aoi_map_to_set_region1 = aoi_map + grass.run_command("v.in.region", output=aoi_map, quiet=True) + else: + aoi_map_mapset = aoi_map.split("@") + aoi_map_to_set_region1 = aoi_map_mapset[0] + if len(aoi_map_mapset) == 2: + mapset = aoi_map_mapset[1] + + # create temporary location with EPSG:25832 + tmp_loc, tmp_gisrc = create_tmp_location(epsg) + + # reproject aoi + if aoi_map: + grass.run_command( + "v.proj", + location=loc, + mapset=mapset, + input=aoi_map_to_set_region1, + output=aoi_map_to_set_region1, + quiet=True, + ) + grass.run_command( + "g.region", + vector=aoi_map_to_set_region1, + res=resolution_to_import, + flags="a", + ) + + # define import parameters + # set memory manually to 1000 + # Process stuck, when memory is too large (100000) + # GDAL_CACHEMAX will be interpreted as MB only + kwargs = { + "input": url, + "output": raster_name, + "memory": 1000, + "quiet": True, + "extent": "region", + } + if fs == "BB_BE": + kwargs["flags"] = "o" + + # download and keep data to download dir if -k flag is set + # and change input parameter in kwargs to local path + if keep_data: + # call download and keep data function for respective federal state + function_name = f"keep_data_{fs}" + if function_name in globals(): + func = globals()[function_name] + kwargs["input"] = func(url, download_dir) + else: + grass.fatal( + _( + f"Function to download and keep data for {fs} not found in lib." + ) + ) + + # import data + import_sucess = False + tries = 0 + while not import_sucess: + tries += 1 + if tries > RETRIES: + grass.fatal( + _( + f"Importing {kwargs['input']} failed after {RETRIES} " + "retries." + ) + ) + try: + grass.run_command("r.import", **kwargs) + import_sucess = True + except Exception: + sleep(WAITING_TIME) + if not aoi_map: + grass.run_command("g.region", raster=f"{raster_name}.1") + + # reproject data + if resolution_to_import: + res = resolution_to_import + else: + res = float( + grass.parse_command("r.info", flags="g", map=f"{raster_name}.1")[ + "nsres" + ] + ) + # switch location + os.environ["GISRC"] = str(gisrc) + if aoi_map: + grass.run_command("g.region", vector=aoi_map, res=res, flags="a") + else: + grass.run_command("g.region", res=res, flags="a") + for i in range(1, 5): + name = f"{raster_name}.{i}" + # set memory manually to 1000 + # Process stuck, when memory is too large (100000) + # GDAL_CACHEMAX is only interpreted as MB, if value is <100000 + grass.run_command( + "r.proj", + location=tmp_loc, + mapset="PERMANENT", + input=name, + output=name, + resolution=res, + flags="n", + quiet=True, + memory=1000, + ) + + # return temp location parameters to remove it in cleanup + return gisdbase, tmp_loc, tmp_gisrc diff --git a/r.dop.import.old/Makefile b/r.dop.import.old/Makefile deleted file mode 100644 index 5f7edfb..0000000 --- a/r.dop.import.old/Makefile +++ /dev/null @@ -1,10 +0,0 @@ -MODULE_TOPDIR = ../.. - -PGM = r.dop.import.old - -ETCFILES = download_urls federal_state_info - -include $(MODULE_TOPDIR)/include/Make/Script.make -include $(MODULE_TOPDIR)/include/Make/Python.make - -default: script diff --git a/r.dop.import.old/download_urls.py b/r.dop.import.old/download_urls.py deleted file mode 100644 index 648e042..0000000 --- a/r.dop.import.old/download_urls.py +++ /dev/null @@ -1,29 +0,0 @@ -URLS = { - "Brandenburg": "https://github.com/mundialis/tile-indices/raw/main/DOP/" - "BE_BB/DOP20_tileindex_BE_BB.gpkg.gz", - "Berlin": "https://github.com/mundialis/tile-indices/raw/main/DOP/BE_BB/" - "DOP20_tileindex_BE_BB.gpkg.gz", - "Baden-Württemberg": None, - "Bayern": None, - "Bremen": None, - "Hessen": None, - "Hamburg": None, - "Mecklenburg-Vorpommern": None, - "Niedersachsen": None, - "Nordrhein-Westfalen": "https://github.com/mundialis/tile-indices/raw/" - "main/DOP/NW/openNRW_DOP10_tileindex.gpkg.gz", - "Rheinland-Pfalz": None, - "Schleswig-Holstein": None, - "Saarland": None, - "Sachsen": "https://github.com/mundialis/tile-indices/raw/main/DOP/SN/" - "DOP20_tileindex_SN.gpkg.gz", - "Sachsen-Anhalt": None, - "Thüringen": None, -} - -WMS_HE = ( - "https://www.gds-srv.hessen.de/cgi-bin/lika-services/ogc-free-" - "images.ows?language=ger&" -) - -WMS_TH = "https://www.geoproxy.geoportal-th.de/geoproxy/services/DOP" diff --git a/r.dop.import.old/federal_state_info.py b/r.dop.import.old/federal_state_info.py deleted file mode 100644 index 390fddd..0000000 --- a/r.dop.import.old/federal_state_info.py +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env python3 -# -############################################################################ -# -# MODULE: federal_state_info -# AUTHOR(S): Anika Weinmann, Julia Haas - -# PURPOSE: German Federal State information as dictionary -# COPYRIGHT: (C) 2024 by mundialis GmbH & Co. KG and the GRASS -# Development Team -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -############################################################################# - -FS_ABBREVIATION = { - "Baden-Württemberg": "BW", - "BW": "BW", - "Bayern": "BY", - "BY": "BY", - "Berlin": "BE", - "BE": "BE", - "Brandenburg": "BB", - "BB": "BB", - "Bremen": "HB", - "HB": "HB", - "Hamburg": "HH", - "HH": "HH", - "Hessen": "HE", - "HE": "HE", - "Mecklenburg-Vorpommern": "MV", - "MV": "MV", - "Niedersachsen": "NI", - "NI": "NI", - "Nordrhein-Westfalen": "NW", - "NW": "NW", - "Rheinland-Pfalz": "RP", - "RP": "RP", - "Saarland": "SL", - "SL": "SL", - "Sachsen": "SN", - "SN": "SN", - "Sachsen-Anhalt": "ST", - "ST": "ST", - "Schleswig-Holstein": "SH", - "SH": "SH", - "Thüringen": "TH", - "TH": "TH", -} diff --git a/r.dop.import.old/install_required_addons.sh b/r.dop.import.old/install_required_addons.sh deleted file mode 100644 index 5698d94..0000000 --- a/r.dop.import.old/install_required_addons.sh +++ /dev/null @@ -1 +0,0 @@ -g.extension extension=r.dop.import.worker url=grass-gis-addons/r.dop.import.worker diff --git a/r.dop.import.old/r.dop.import.old.html b/r.dop.import.old/r.dop.import.old.html deleted file mode 100644 index a0ed62a..0000000 --- a/r.dop.import.old/r.dop.import.old.html +++ /dev/null @@ -1,103 +0,0 @@ -

DESCRIPTION

- -r.dop.import.old downloads digital orthophotos (DOPs) and imports them -into GRASS. Alternatively, local data can be imported. -The output will be four raster maps: <output>_red, <output>_green, -<output>_blue, <output>_nir, while each raster map displays one virtual -raster, combining all DOP-tiles, and <output> is given by output -option. - -Note, that the value range is modified from [0 255] to [1 256]. -To download the data, the federal_state(s) of the area of interest has to be given. -Either given in a text file, comma-separated for multiple federal_states (filepath option). -Or given directly as input, comma-separated for multiple federal_states (federal_state option). -It is only used to get the corresponding download link and is NOT used as an alternative area of interset. -By default the data are imported for the current region. With the option aoi_map, some vector map in GRASS, the data are imported only for the given AOI. -The resolution is set by default to the resolution of the region. With the -r flag the original available resolution of the DOPs is used. -The DOPs are imported in parallel in temp-mapsets. The final computed maps are then copied from the temp-mapset to the current mapset. -The number of parallel processes is given by nprocs. - -Implemented federal state options are: -
    -
  • Baden-Würrtemberg: only local data
    • -
  • Brandenburg. Native resolution: 0.2m
    • -
  • Hessen. Native resolution: 0.2m
    • -
  • Nordrhein-Westfalen. Native resolution: 0.1m
    • -
  • Sachsen. Native resolution: 0.2m (gets imported as 0.18m and is resampled to 0.2m if -r flag is set)
    • -
  • Thüringen. Native resolution: 0.2m
    • -
- -For local data import the parameter local_data_dir has to be given and -the folder structure has to be as follows: -
-/path/to/DOPs/
-├── BW/*.vrt
-└── NW/*.tif
-└── ...
-
-In the federal state folders the addons searches for a vrt file or -if none is given all tifs will be imported. -If local data does not overlap with AOI, data will be downloaded from Open Data -portals if federal state supports Open Data. - -

EXAMPLE

- -

Load DOPs for region with federal state information given by -federal_state option

- -
-r.dop.import.old output=dops federal_state=Nordrhein-Westfalen
-
- -

Load DOPs for region with 4 parallel processes

- -
-r.dop.import.old output=dops federal_state=Nordrhein-Westfalen nprocs=4
-
- -

Load DOPs for region with federal state information given by -filepath option

- -
-v.check.federal_state aoi=polygon_aoi federal_state_file=federal_state.txt
-r.dop.import.old output=dops filepath=federal_state.txt
-
- -With file content of federal_state.txt: - -
-Berlin,Brandenburg
-
- -

Load DOPs for AOI given by aoi_map_example

- -
-r.dop.import.old output=dops federal_state=Nordrhein-Westfalen aoi_map=aoi_map_example
-
- -

Load DOPs for AOI given by aoi_map_example and with original DOPs resolution

- -
-r.dop.import.old output=dops federal_state=Nordrhein-Westfalen aoi_map=aoi_map_example -r
-
- -

Use local DOPs

-Use local data instead of open data for NRW: -
-r.dop.import.old output=dops federal_state=Nordrhein-Westfalen \
-    aoi_map=aoi_map_example local_data_dir=/path/to/DOPs/
-
- - -

SEE ALSO

- -r.dop.import.worker, -r.buildvrt, -r.import, -v.check.federal_state - - -

AUTHORS

-Johannes Halbauer, mundialis GmbH & Co. KG -Lina Krisztian, mundialis GmbH & Co. KG -Anika Weinmann, mundialis GmbH & Co. KG diff --git a/r.dop.import.old/r.dop.import.old.py b/r.dop.import.old/r.dop.import.old.py deleted file mode 100644 index d2a74cb..0000000 --- a/r.dop.import.old/r.dop.import.old.py +++ /dev/null @@ -1,803 +0,0 @@ -#!/usr/bin/env python3 -# -############################################################################ -# -# MODULE: r.dop.import.old -# AUTHOR(S): Johannes Halbauer, Lina Krisztian, Anika Weinmann, Julia Haas -# -# PURPOSE: Downloads Digital Orthophotos (DOPs) within a specified area -# (currently only for NRW) -# COPYRIGHT: (C) 2022-2024 by mundialis GmbH & Co. KG and the GRASS -# Development Team -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -############################################################################# - -# %Module -# % description: Downloads and imports Digital Othophotos (DOPs) (currently only for NRW) -# % keyword: imagery -# % keyword: download -# % keyword: DOP -# %end - -# %option G_OPT_R_OUTPUT -# % key: output -# % required: yes -# %end - -# %option G_OPT_V_INPUT -# % key: aoi_map -# % required: no -# % description: Vector map to restrict DOPs import to -# %end - -# %option -# % key: filepath -# % required: no -# % description: Text file containing federal state to load DOPs for -# %end - -# %option -# % key: federal_state -# % multiple: yes -# % required: no -# % description: Federal state to load DOPs for (no alternative to aoi_map; parameter is required/used for getting download-URL only) -# % options: Brandenburg,Berlin,Baden-Württemberg,Bayern,Bremen,Hessen,Hamburg,Mecklenburg-Vorpommern,Niedersachsen,Nordrhein-Westfalen,Rheinland-Pfalz,Schleswig-Holstein,Saarland,Sachsen,Sachsen-Anhalt,Thüringen -# %end - -# %option G_OPT_M_DIR -# % key: local_data_dir -# % required: no -# % description: Directory with raster map of DOPs to import (e.g. VRT) -# %end - -# %option -# % key: nprocs -# % type: integer -# % required: no -# % multiple: no -# % label: Number of parallel processes -# % description: Number of cores for multiprocessing, -2 is the number of available cores - 1 -# % answer: -2 -# %end - -# %flag -# % key: r -# % description: use native DOP resolution -# %end - -# %rules -# % required: federal_state, filepath -# % excludes: filepath, federal_state -# %end - -import atexit -import glob -import os -import wget -import multiprocessing as mp -import grass.script as grass -from grass.pygrass.modules import Module, ParallelModuleQueue -import sys -from grass_gis_helpers.cleanup import general_cleanup -from grass_gis_helpers.general import communicate_grass_command - - -sys.path.insert( - 1, - os.path.join( - os.path.dirname(sys.path[0]), - "etc", - "r.dop.import.old", - ), -) -from download_urls import URLS -from download_urls import WMS_HE -from download_urls import WMS_TH -from federal_state_info import FS_ABBREVIATION - -tmp_dir = None -resolution_to_import = None -rm_vectors = [] -rm_rasters = [] -rm_groups = [] -orig_region = None -mapset_names = [] - - -def cleanup(): - rm_dirs = [] - if tmp_dir: - rm_dirs.append(tmp_dir) - general_cleanup( - rm_vectors=rm_vectors, - rm_rasters=rm_rasters, - rm_groups_wo_rasters=rm_groups, - orig_region=orig_region, - rm_dirs=rm_dirs, - ) - # remove mapsets - for rm_mapset in mapset_names: - gisenv = grass.gisenv() - mapset_path = os.path.join( - gisenv["GISDBASE"], gisenv["LOCATION_NAME"], rm_mapset - ) - grass.try_rmdir(mapset_path) - - -def setup_parallel_processing(nprocs): - if nprocs == -2: - nprocs = mp.cpu_count() - 1 if mp.cpu_count() > 1 else 1 - else: - # Test nprocs settings - nprocs_real = mp.cpu_count() - if nprocs > nprocs_real: - grass.warning( - "Using %d parallel processes but only %d CPUs available." - % (nprocs, nprocs_real) - ) - - # set some common environmental variables, like: - os.environ.update( - dict( - GRASS_COMPRESSOR="LZ4", - GRASS_MESSAGE_FORMAT="plain", - ) - ) - return nprocs - - -def reset_region(region): - """Function to set the region to the given region - Args: - region (str): the name of the saved region which should be set and - deleted - """ - nulldev = open(os.devnull, "w") - kwargs = {"flags": "f", "quiet": True, "stderr": nulldev} - if region: - if grass.find_file(name=region, element="windows")["file"]: - grass.run_command("g.region", region=region) - grass.run_command("g.remove", type="region", name=region, **kwargs) - - -def create_grid(tile_size, grid_prefix, area): - """Create a grid for parallelization - Args: - tile_size (float): the size for the tiles in map units - grid_prefix (str): the prefix name for the output grid - area (str): the name of area for which to create the grid tiles - Return: - tiles_list (list): list with the names of the created vector map tiles - """ - if area is None or area == "": - area = f"tmp_aoi_{grass.tempname(8)}" - rm_vectors.append(area) - grass.run_command("v.in.region", output=area) - # set region to area - region = grass.parse_command("g.region", flags="ug", vector=area) - dist_ns = abs(float(region["n"]) - float(region["s"])) - dist_ew = abs(float(region["w"]) - float(region["e"])) - - grass.message(_("Creating tiles...")) - grid = f"tmp_grid_{grass.tempname(8)}" - # check if region is smaller than tile size - if dist_ns <= float(tile_size) and dist_ew <= float(tile_size): - grass.run_command( - "g.region", vector=area, res=resolution_to_import, flags="a" - ) - grass.run_command("v.in.region", output=grid, quiet=True) - grass.run_command( - "v.db.addtable", map=grid, columns="cat int", quiet=True - ) - else: - # set region - orig_region = f"grid_region_{grass.tempname(8)}" - grass.run_command("g.region", save=orig_region, quiet=True) - grass.run_command("g.region", vector=area, quiet=True) - grass.run_command("g.region", res=tile_size, flags="a", quiet=True) - - # create grid - grass.run_command( - "v.mkgrid", map=grid, box=f"{tile_size},{tile_size}", quiet=True - ) - # reset region - reset_region(orig_region) - grid_name = f"tmp_grid_area_{grass.tempname(8)}" - grass.run_command( - "v.select", - ainput=grid, - binput=area, - output=grid_name, - operator="overlap", - quiet=True, - ) - if grass.find_file(name=grid_name, element="vector")["file"] == "": - grass.fatal( - _( - f"The set region is not overlapping with {area}. " - f"Please define another region." - ) - ) - - # create list of tiles - tiles_num_list = list( - grass.parse_command( - "v.db.select", map=grid_name, columns="cat", flags="c", quiet=True - ).keys() - ) - - number_tiles = len(tiles_num_list) - grass.message(_(f"Number of tiles is: {number_tiles}")) - tiles_list = [] - for tile in tiles_num_list: - tile_area = f"{grid_prefix}_{tile}" - grass.run_command( - "v.extract", - input=grid_name, - where=f"cat == {tile}", - output=tile_area, - quiet=True, - ) - tiles_list.append(tile_area) - rm_vectors.append(tile_area) - - # cleanup - nuldev = open(os.devnull, "w") - kwargs = {"flags": "f", "quiet": True, "stderr": nuldev} - for rmv in [grid, grid_name]: - if grass.find_file(name=rmv, element="vector")["file"]: - grass.run_command("g.remove", type="vector", name=rmv, **kwargs) - - return tiles_list - - -def get_tindex(tileindex): - """Download and import tindex - Args: - tileindex ... URL to tile index - Returns: - vm_import ... Name of the tile index vector map - """ - # download tindex - zipname = os.path.basename(tileindex) - tmp_dir = grass.tempdir() - download_path = os.path.join(tmp_dir, zipname) - wget.download(tileindex, download_path, bar=None) - - # unzip tindex - unzipped_name = zipname.replace(".gz", "") - unzipped_path = os.path.join(tmp_dir, unzipped_name) - os.system(f"gunzip {download_path}") - - # import vector map containing URL for each tile - vm_import = f"vm_import_{grass.tempname(8)}" - grass.run_command( - "v.import", - input=unzipped_path, - output=vm_import, - extent="region", - overwrite=True, - quiet=True, - ) - rm_vectors.append(vm_import) - return vm_import - - -def download_and_clip_tindex(federal_state, aoi_map=None): - """Download and clip tindex - Args: - aoi_map (str): name of AOI vector map - Returns: - (list): list with urls of tiles - """ - tileindex = URLS[federal_state] - if tileindex is None: - grass.warning(_(f"{federal_state} is not yet implemented.")) - return [] - else: - # if aoi_map given: set region to aoi_map extent - if aoi_map: - grass.run_command( - "g.region", - vector=aoi_map, - res=resolution_to_import, - flags="a", - quiet=True, - ) - - # download and unzip tile index - vm_import = get_tindex(tileindex) - if not grass.find_file(name=vm_import, element="vector")["file"]: - grass.fatal( - _( - "No tile found in region. Please check if region or aoi " - "overlap with federal state" - ) - ) - if not aoi_map: - aoi_map = f"aoi_map_{grass.tempname(8)}" - rm_vectors.append(aoi_map) - grass.run_command("v.in.region", output=aoi_map) - # check which tiles are needed for selected AOI - vm_clip = f"vm_clip_{grass.tempname(8)}" - grass.run_command( - "v.clip", - input=vm_import, - clip=aoi_map, - output=vm_clip, - flags="d", - overwrite=True, - quiet=True, - ) - rm_vectors.append(vm_clip) - # else: - # # if no aoi given, use complete (current set) region: - # vm_clip = vm_import - - # import tiles and rename them according to their band - # and write them in a list - return grass.vector_db_select(vm_clip, columns="location")[ - "values" - ].items() - - -def get_tiles(federal_state, aoi_map=None): - """Get or create tileindex for federal state - Args: - federal_state (str): A string with a federal state - aoi_map (str): Name of AOI vector map - Returns: - tileindex: None if no tileindex exists and one was created - tiles_list (list): list of tiles (names of the created vector tiles or - URLs to download DOPs) - """ - if federal_state in URLS: - grass.message(f"Processing {federal_state}...") - if federal_state in ["Hessen", "Thüringen"]: - # create grid for wms import - if federal_state == "Hessen": - tiles_list = create_grid(1000, "HE_DOP", aoi_map) - elif federal_state == "Thüringen": - tiles_list = create_grid(1000, "TH_DOP", aoi_map) - # no tileindex - tileindex = None - - else: - tileindex = URLS[federal_state] - if tileindex is None: - grass.warning(_(f"{federal_state} is not yet implemented.")) - tiles_list = [] - else: - grass.message(_("Import tindex ...")) - tiles_list = download_and_clip_tindex(federal_state, aoi_map) - else: - if options["filepath"]: - grass.fatal( - _( - "Non valid name of federal state," - " in 'filepath'-option given" - ) - ) - elif options["federal_state"]: - grass.fatal( - _( - "Non valid name of federal state," - " in 'federal_states'-option given" - ) - ) - return tileindex, tiles_list - - -def adjust_raster_resolution(raster_name, output, res): - """Resample or inpolate raster to given resolution""" - res_rast = float( - grass.parse_command("r.info", map=raster_name, flags="g")["nsres"] - ) - if res_rast > res: - grass.run_command( - "r.resamp.interp", - input=raster_name, - output=output, - overwrite=True, - quiet=True, - ) - elif res_rast < res: - grass.run_command( - "r.resamp.stats", - input=raster_name, - output=output, - method="median", - quiet=True, - overwrite=True, - ) - else: - rename_raster(raster_name, output) - - -def adjust_resolution_for_rasters(raster_base_name): - """Resample or inpolate raster for all bands""" - res = resolution_to_import - # set region to imported raster - grass.run_command("g.region", raster=f"{raster_base_name}.1", quiet=True) - grass.run_command("g.region", res=res, flags="a", quiet=True) - for band in range(1, 5): - adjust_raster_resolution( - f"{raster_base_name}.{band}", f"{raster_base_name}.{band}", res - ) - - -def create_vrt(b_list, out): - # copy raster maps to current mapset - for rast in b_list: - if "@" in rast: - rast_wo_mapsetname = rast.split("@")[0] - grass.run_command( - "g.copy", - raster=f"{rast},{rast_wo_mapsetname}", - ) - b_list = [val.split("@")[0] for val in b_list] - # buildvrt if required + renaming to output name - if len(b_list) > 1: - grass.run_command("g.region", raster=b_list) - grass.run_command( - "r.buildvrt", input=b_list, output=out, quiet=True, overwrite=True - ) - else: - grass.run_command( - "g.rename", raster=f"{b_list[0]},{out}", quiet=True, overwrite=True - ) - - -def download_and_import_dops( - aoi, - fs, - federal_state, - resolution_to_import, - rm_rasters, - orig_region, - mapset_names, - get_tiles, - all_raster, - nprocs, -): - tileindex, tiles_list = get_tiles(federal_state, aoi) - number_tiles = len(tiles_list) - # set number of parallel processes to number of tiles - # if multiple federal states given, - # they are not calculated in parallel so far - if number_tiles < nprocs: - nprocs = number_tiles - queue = ParallelModuleQueue(nprocs=nprocs) - try: - grass.message(_(f"Importing {len(tiles_list)} DOPs in parallel...")) - for tile_el in tiles_list: - if tileindex: - key = tile_el[0] - new_mapset = f"tmp_mapset_rdop_import_tile_{key}_{os.getpid()}" - mapset_names.append(new_mapset) - b_name = os.path.basename(tile_el[1][0]) - raster_name = ( - f"{b_name.split('.')[0].replace('-', '_')}" - f"_{os.getpid()}" - ) - for key_rast in all_raster: - all_raster[key_rast].append( - f"{fs}_{raster_name}_{key_rast}@{new_mapset}" - ) - param = { - "flags": "t", - "tile_key": key, - "tile_url": tile_el[1][0], - "federal_state": fs, - "raster_name": raster_name, - "orig_region": orig_region, - "memory": 1000, - "new_mapset": new_mapset, - } - else: - key = tile_el - new_mapset = f"tmp_mapset_rdop_import_tile_{key}_{os.getpid()}" - mapset_names.append(new_mapset) - raster_name = tile_el - for key_rast in all_raster: - all_raster[key_rast].append( - f"{fs}_{raster_name}_{key_rast}@{new_mapset}" - ) - if fs == "HE": - wms = WMS_HE - elif fs == "TH": - wms = WMS_TH - param = { - "flags": "", - "tile_key": key, - "tile_url": wms, - "federal_state": fs, - "raster_name": raster_name, - "orig_region": orig_region, - "memory": 1000, - "new_mapset": new_mapset, - } - grass.message(f"raster_name: {raster_name}") - if aoi: - param["aoi_map"] = aoi - - if flags["r"]: - param["flags"] += "r" - else: - param["resolution_to_import"] = resolution_to_import - # add downloaded raster bands to rm_rast - rm_red = f"{raster_name}_red" - rm_green = f"{raster_name}_green" - rm_blue = f"{raster_name}_blue" - rm_rasters.append(rm_red) - rm_rasters.append(rm_green) - rm_rasters.append(rm_blue) - # grass.run_command( - r_dop_import_worker = Module( - "r.dop.import.worker", - **param, - run_=False, - ) - # catch all GRASS outputs to stdout and stderr - r_dop_import_worker.stdout_ = grass.PIPE - r_dop_import_worker.stderr_ = grass.PIPE - queue.put(r_dop_import_worker) - queue.wait() - except Exception: - for proc_num in range(queue.get_num_run_procs()): - proc = queue.get(proc_num) - if proc.returncode != 0: - # save all stderr to a variable and pass it to a GRASS - # exception - errmsg = proc.outputs["stderr"].value.strip() - grass.fatal( - _(f"\nERROR by processing <{proc.get_bash()}>: {errmsg}") - ) - - -def rename_raster(band_name_old, band_name_new): - """Rename raster map""" - grass.run_command( - "g.rename", - raster=f"{band_name_old},{band_name_new}", - quiet=True, - overwrite=True, - ) - - -def import_local_data(aoi, local_data_dir, fs, all_raster): - """Import of data from local file path - Args: - aoi (str): name of vector map defining AOI - local_data_dir (str): path to local data - fs (str): federal state abbreviation - all_raster (dict): dictionary with bands and output raster list - Returns: - imported_local_data (bool): True if local data imported, otherwise False - """ - grass.message(_("Importing local data...")) - imported_local_data = False - # get files (VRT if available otherwise tif) - dop_files = glob.glob( - os.path.join(local_data_dir, fs, "**", "*.vrt"), - recursive=True, - ) - if not dop_files: - dop_files = glob.glob( - os.path.join(local_data_dir, fs, "**", "*.tif"), - recursive=True, - ) - - # import data for AOI - # TODO parallize local data import for multi tifs - for i, dop_file in enumerate(dop_files): - cur_reg = grass.region() - ns_res = cur_reg["nsres"] - ew_res = cur_reg["ewres"] - if aoi and aoi != "": - grass.run_command( - "g.region", - vector=aoi, - nsres=ns_res, - ewres=ew_res, - flags="a", - quiet=True, - ) - name = f"dop_{i}" - kwargs = { - "input": dop_file, - "output": name, - "extent": "region", - "quiet": True, - "overwrite": True, - } - # resolution settings: -r native resolution; otherwise from region - if not flags["r"]: - kwargs["resolution"] = "region" - r_import = communicate_grass_command("r.import", **kwargs) - err_m1 = "Input raster does not overlap current computational region." - err_m2 = "already exists and will be overwritten" - if err_m1 in r_import[1].decode(): - continue - elif err_m2 in r_import[1].decode(): - pass - elif r_import[1].decode() != "": - grass.fatal(_(r_import[1].decode())) - - band_dict = {1: "red", 2: "green", 3: "blue", 4: "nir"} - for band_num, band in band_dict.items(): - band_name_old = f"{name}.{band_num}" - band_name_new = f"{name}.{band}" - rm_rasters.append(band_name_old) - # check resolution and resample if needed - if not flags["r"]: - adjust_raster_resolution(band_name_old, band_name_new, ns_res) - else: - rename_raster(band_name_old, band_name_new) - all_raster[band].append(band_name_new) - - # check if all bands have at least one input raster - if all([len(val) > 0 for band, val in all_raster.items()]): - imported_local_data = True - - if not imported_local_data and fs in ["BW"]: - grass.fatal(_("Local data does not overlap with AOI.")) - elif not imported_local_data: - grass.message( - _( - "Local data does not overlap with AOI. Data will be downloaded" - " from Open Data portal." - ) - ) - return imported_local_data - - -def main(): - global tmp_dir, orig_region, rm_vectors, rm_rasters, rm_groups - global resolution_to_import, mapset_names - - # a vector map, consisting of the DOP-tiles, - # while each DOP-tile contains its corresponding - # download link. - # By overlaying the aoi_map with this vector map - # the download links of the DOPs within the area - # of interest are selected. - # However, when the aoi_map lies exactly at the border - # of such DOP tile, a problem due to floating point - # precision limit can occur. This results in a larger - # overlay-area of aoi and vector map, than it is - # actually the case. - # Two approaches for solution: - # 1. modify the gpkg-file: delete the decimals - # (this avoids the floating proint precision problem) - # 2. do not use the gpkg-file; instead generate the - # DOP download links via the UTM-coordinates - - # parser options - aoi = options["aoi_map"] - local_data_dir = options["local_data_dir"] - nprocs = int(options["nprocs"]) - nprocs = setup_parallel_processing(nprocs) - - # check if required addons installed: - addon = "r.dop.import.worker" - if not grass.find_program(addon, "--help"): - msg = ( - f"The '{addon}' module was not found, install it first:\n" - f"g.extension {addon}" - ) - grass.fatal(_(msg)) - - # create list for each raster band for building entire raster - # of all given federal states - all_raster = { - "red": [], - "green": [], - "blue": [], - "nir": [], - } - - # check if overwrite is not activated - if os.getenv("GRASS_OVERWRITE", "0") != "1": - # check if output raster maps already exists - for band in all_raster: - out = f"{options['output']}_{band}" - if grass.find_file(name=out, element="raster")["file"]: - grass.fatal( - _( - f"output map <{out}> exists. To overwrite, use the " - "--overwrite flag" - ) - ) - - # read federal state(s) from input options - if options["filepath"]: - with open(f'{options["filepath"]}') as f: - federal_states = f.read().strip().split(",") - else: - federal_states = options["federal_state"].split(",") - - # get list of local input folders for federal states - local_fs_list = [] - if local_data_dir and local_data_dir != "": - local_fs_list = os.listdir(local_data_dir) - - # Berlin and Brandenburg are using the same wms of BB - if "Berlin" in federal_states and "Brandenburg" in federal_states: - BE_idx = federal_states.index("Berlin") - del federal_states[BE_idx] - - # save current region for setting back later in cleanup - orig_region = f"orig_region_{grass.tempname(8)}" - grass.run_command("g.region", save=orig_region, overwrite=True, quiet=True) - reg = grass.region() - if flags["r"]: - # TODO: event auf None - resolution_to_import = 0.2 - else: - if reg["nsres"] == reg["ewres"]: - resolution_to_import = float(reg["nsres"]) - else: - grass.fatal("N/S resolution is not the same as E/W resolution!") - - # loop through federal states and get respective DOPs - for federal_state in federal_states: - if federal_state not in FS_ABBREVIATION: - grass.fatal(_(f"Non valid name of federal state: {federal_state}")) - fs = FS_ABBREVIATION[federal_state] - - # check if local data for federal state given - imported_local_data = False - if fs in local_fs_list: - # TODO import_local_data with -r flag - imported_local_data = import_local_data( - aoi, local_data_dir, fs, all_raster - ) - elif fs in ["BW"]: - grass.fatal( - _(f"No local data for {fs} available. Is the path correct?") - ) - - # import data when local import was not used - if not imported_local_data: - # check if federal state is supported - if fs in ["BE", "BB", "HE", "NW", "SN", "TH"]: - # import data - download_and_import_dops( - aoi, - fs, - federal_state, - resolution_to_import, - rm_rasters, - orig_region, - mapset_names, - get_tiles, - all_raster, - nprocs, - ) - else: - grass.warning(_(f"Support for {fs} is not yet implemented.")) - - raster_out = [] - for band, b_list in all_raster.items(): - out = f"{options['output']}_{band}" - create_vrt(b_list, out) - raster_out.append(out) - - grass.message(_(f"Generated following raster maps: {raster_out}")) - - -if __name__ == "__main__": - options, flags = grass.parser() - atexit.register(cleanup) - main() diff --git a/r.dop.import.worker.bb.be/r.dop.import.worker.bb.be.py b/r.dop.import.worker.bb.be/r.dop.import.worker.bb.be.py index ec67a28..4f35a59 100644 --- a/r.dop.import.worker.bb.be/r.dop.import.worker.bb.be.py +++ b/r.dop.import.worker.bb.be/r.dop.import.worker.bb.be.py @@ -5,8 +5,10 @@ # MODULE: r.dop.import.worker.bb.be # AUTHOR(S): Johannes Halbauer & Lina Krisztian # -# PURPOSE: Downloads Digital Orthophotos (DOPs) within a specified area in Brandenburg -# COPYRIGHT: (C) 2024 by mundialis GmbH & Co. KG and the GRASS Development Team +# PURPOSE: Downloads Digital Orthophotos (DOPs) within a specified area +# in Brandenburg +# COPYRIGHT: (C) 2024 by mundialis GmbH & Co. KG and the GRASS Development +# Team # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -94,25 +96,15 @@ import atexit -import os import sys -from time import sleep import grass.script as grass from grass.pygrass.utils import get_lib_path from grass_gis_helpers.cleanup import general_cleanup, cleaning_tmp_location from grass_gis_helpers.general import test_memory -from grass_gis_helpers.location import ( - get_current_location, - create_tmp_location, - switch_back_original_location, -) +from grass_gis_helpers.location import switch_back_original_location from grass_gis_helpers.mapset import switch_to_new_mapset -from grass_gis_helpers.open_geodata_germany.download_data import ( - download_data_using_threadpool, - extract_compressed_files, -) from grass_gis_helpers.raster import adjust_raster_resolution # import module library @@ -121,7 +113,7 @@ grass.fatal("Unable to find the dop library directory.") sys.path.append(path) try: - from r_dop_import_lib import rescale_to_1_256 + from r_dop_import_lib import rescale_to_1_256, import_and_reproject except Exception as imp_err: grass.fatal(f"r.dop.import library could not be imported: {imp_err}") @@ -132,9 +124,6 @@ tmp_loc = None tmp_gisrc = None -RETRIES = 30 -WAITING_TIME = 10 - def cleanup(): cleaning_tmp_location( @@ -146,143 +135,9 @@ def cleanup(): ) -def import_and_reproject( - url, - raster_name, - resolution_to_import, - aoi_map=None, - download_dir=None, - epsg=None, -): - """Import DOPs and reproject them if needed. This is needed for the DOPs - of Brandenburg and Berlin because GDAL (at least smaller 3.6.3) does not - support the coordinate reference system in the data. - - Args: - url (str): The URL of the DOP to import - raster_name (str): The prefix name for the output rasters - aoi_map (str): Name of AOI vector map - download_dir (str): Path to local directory to downlaod DOPs to - epsg (int): EPSG code which has to be set if the reproduction should be - done manually and not by r.import - """ +def main(): global gisdbase, tmp_loc, tmp_gisrc - aoi_map_to_set_region1 = aoi_map - - # get actual location, mapset, ... - loc, mapset, gisdbase, gisrc = get_current_location() - if not aoi_map: - aoi_map = f"region_aoi_{grass.tempname(8)}" - aoi_map_to_set_region1 = aoi_map - grass.run_command("v.in.region", output=aoi_map, quiet=True) - else: - aoi_map_mapset = aoi_map.split("@") - aoi_map_to_set_region1 = aoi_map_mapset[0] - if len(aoi_map_mapset) == 2: - mapset = aoi_map_mapset[1] - - # create temporary location with EPSG:25833 - tmp_loc, tmp_gisrc = create_tmp_location(epsg) - - # reproject aoi - if aoi_map: - grass.run_command( - "v.proj", - location=loc, - mapset=mapset, - input=aoi_map_to_set_region1, - output=aoi_map_to_set_region1, - quiet=True, - ) - grass.run_command( - "g.region", - vector=aoi_map_to_set_region1, - res=resolution_to_import, - flags="a", - ) - # import data - # set memory manually to 1000 - # Process stuck, when memory is too large (100000) - # GDAL_CACHEMAX wird nur als MB interpretiert - kwargs = { - "input": url, - "output": raster_name, - "memory": 1000, - "quiet": True, - "flags": "o", - "extent": "region", - } - - # TODO: Auslagern dieser Funktion in Lib, jedoch unterscheidet - # sich der folgende if-Block bspw. von dem von NW - # (außerdem ist noch die o-flag in kwargs unterschiedlich) - # - # download and keep data to download dir if -k flag ist set - # and change input parameter in kwargs to local path - if flags["k"]: - url = url.replace("/vsizip/vsicurl/", "") - basename = os.path.basename(url) - url = url.replace(basename, "")[:-1] - download_data_using_threadpool([url], download_dir, 1) - extract_compressed_files( - [basename.replace(".tif", ".zip")], download_dir - ) - kwargs["input"] = os.path.join(download_dir, basename) - - import_sucess = False - tries = 0 - while not import_sucess: - tries += 1 - if tries > RETRIES: - grass.fatal( - _( - f"Importing {kwargs['input']} failed after {RETRIES} " - "retries." - ) - ) - try: - grass.run_command("r.import", **kwargs) - import_sucess = True - except Exception: - sleep(WAITING_TIME) - if not aoi_map: - grass.run_command("g.region", raster=f"{raster_name}.1") - - # reproject data - if resolution_to_import: - res = resolution_to_import - else: - res = float( - grass.parse_command("r.info", flags="g", map=f"{raster_name}.1")[ - "nsres" - ] - ) - # switch location - os.environ["GISRC"] = str(gisrc) - if aoi_map: - grass.run_command("g.region", vector=aoi_map, res=res, flags="a") - else: - grass.run_command("g.region", res=res, flags="a") - for i in range(1, 5): - name = f"{raster_name}.{i}" - # set memory manually to 1000 - # Process stuck, when memory is too large (100000) - # GDAL_CACHEMAX is only interpreted as MB, if value is <100000 - grass.run_command( - "r.proj", - location=tmp_loc, - mapset="PERMANENT", - input=name, - output=name, - resolution=res, - flags="n", - quiet=True, - memory=1000, - ) - - -def main(): # parser options tile_key = options["tile_key"] tile_url = options["tile_url"] @@ -291,6 +146,7 @@ def main(): orig_region = options["orig_region"] new_mapset = options["new_mapset"] download_dir = options["download_dir"] + keep_data = flags["k"] # set memory to input if possible options["memory"] = test_memory(options["memory"]) @@ -311,13 +167,15 @@ def main(): ) # import and reproject DOP tiles based on tileindex - import_and_reproject( + gisdbase, tmp_loc, tmp_gisrc = import_and_reproject( tile_url, raster_name, resolution_to_import, + "BB_BE", aoi_map, download_dir, epsg=25833, + keep_data=keep_data, ) # adjust resolution if required diff --git a/r.dop.import.worker.he/r.dop.import.worker.he.py b/r.dop.import.worker.he/r.dop.import.worker.he.py index 2d71236..21b8676 100644 --- a/r.dop.import.worker.he/r.dop.import.worker.he.py +++ b/r.dop.import.worker.he/r.dop.import.worker.he.py @@ -5,8 +5,10 @@ # MODULE: r.dop.import.worker.he # AUTHOR(S): Johannes Halbauer & Lina Krisztian # -# PURPOSE: Downloads Digital Orthophotos (DOPs) within a specified area in Hessen -# COPYRIGHT: (C) 2024 by mundialis GmbH & Co. KG and the GRASS Development Team +# PURPOSE: Downloads Digital Orthophotos (DOPs) within a specified area +# in Hessen +# COPYRIGHT: (C) 2024 by mundialis GmbH & Co. KG and the GRASS Development +# Team # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/r.dop.import.worker.nw/r.dop.import.worker.nw.py b/r.dop.import.worker.nw/r.dop.import.worker.nw.py index 59e01fe..36c8455 100644 --- a/r.dop.import.worker.nw/r.dop.import.worker.nw.py +++ b/r.dop.import.worker.nw/r.dop.import.worker.nw.py @@ -5,8 +5,10 @@ # MODULE: r.dop.import.worker.nw # AUTHOR(S): Johannes Halbauer & Lina Krisztian # -# PURPOSE: Downloads Digital Orthophotos (DOPs) within a specified area in Nordrhein-Westfalen -# COPYRIGHT: (C) 2024 by mundialis GmbH & Co. KG and the GRASS Development Team +# PURPOSE: Downloads Digital Orthophotos (DOPs) within a specified area +# in Nordrhein-Westfalen +# COPYRIGHT: (C) 2024 by mundialis GmbH & Co. KG and the GRASS Development +# Team # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -99,24 +101,15 @@ import atexit -import os import sys -from time import sleep import grass.script as grass from grass.pygrass.utils import get_lib_path from grass_gis_helpers.cleanup import general_cleanup, cleaning_tmp_location from grass_gis_helpers.general import test_memory -from grass_gis_helpers.location import ( - get_current_location, - create_tmp_location, - switch_back_original_location, -) +from grass_gis_helpers.location import switch_back_original_location from grass_gis_helpers.mapset import switch_to_new_mapset -from grass_gis_helpers.open_geodata_germany.download_data import ( - download_data_using_threadpool, -) from grass_gis_helpers.raster import adjust_raster_resolution # import module library @@ -125,7 +118,7 @@ grass.fatal("Unable to find the dop library directory.") sys.path.append(path) try: - from r_dop_import_lib import rescale_to_1_256 + from r_dop_import_lib import rescale_to_1_256, import_and_reproject except Exception as imp_err: grass.fatal(f"r.dop.import library could not be imported: {imp_err}") @@ -136,9 +129,6 @@ tmp_loc = None tmp_gisrc = None -RETRIES = 30 -WAITING_TIME = 10 - def cleanup(): cleaning_tmp_location( @@ -150,133 +140,9 @@ def cleanup(): ) -def import_and_reproject( - url, - raster_name, - resolution_to_import, - aoi_map=None, - download_dir=None, - epsg=None, -): - """Import DOPs and reproject them if needed. - - Args: - url (str): The URL of the DOP to import - raster_name (str): The prefix name for the output rasters - aoi_map (str): Name of AOI vector map - download_dir (str): Path to local directory to downlaod DOPs to - epsg (int): EPSG code which has to be set if the reproduction should be - done manually and not by r.import - """ +def main(): global gisdbase, tmp_loc, tmp_gisrc - aoi_map_to_set_region1 = aoi_map - - # get actual location, mapset, ... - loc, mapset, gisdbase, gisrc = get_current_location() - if not aoi_map: - aoi_map = f"region_aoi_{grass.tempname(8)}" - aoi_map_to_set_region1 = aoi_map - grass.run_command("v.in.region", output=aoi_map, quiet=True) - else: - aoi_map_mapset = aoi_map.split("@") - aoi_map_to_set_region1 = aoi_map_mapset[0] - if len(aoi_map_mapset) == 2: - mapset = aoi_map_mapset[1] - - # create temporary location with EPSG:25832 - tmp_loc, tmp_gisrc = create_tmp_location(epsg) - - # reproject aoi - if aoi_map: - grass.run_command( - "v.proj", - location=loc, - mapset=mapset, - input=aoi_map_to_set_region1, - output=aoi_map_to_set_region1, - quiet=True, - ) - grass.run_command( - "g.region", - vector=aoi_map_to_set_region1, - res=resolution_to_import, - flags="a", - ) - - # import data - # set memory manually to 1000 - # Process stuck, when memory is too large (100000) - # GDAL_CACHEMAX wird nur als MB interpretiert - kwargs = { - "input": url, - "output": raster_name, - "memory": 1000, - "quiet": True, - # "flags": "o", - "extent": "region", - } - # TODO: Auslagern dieser Funktion in Lib, jedoch unterscheidet - # sich der folgende if-Block bspw. von dem von BB/BE - # (außerdem ist noch die o-flag in kwargs unterschiedlich) - # - # download and keep data to download dir if -k flag ist set - # and change input parameter in kwargs to local path - if flags["k"]: - url = url.replace("/vsicurl/", "") - basename = os.path.basename(url) - download_data_using_threadpool([url], download_dir, 1) - kwargs["input"] = os.path.join(download_dir, basename) - - import_sucess = False - tries = 0 - while not import_sucess: - tries += 1 - if tries > RETRIES: - grass.fatal( - _( - f"Importing {kwargs['input']} failed after {RETRIES} " - "retries." - ) - ) - try: - grass.run_command("r.import", **kwargs) - import_sucess = True - except Exception: - sleep(WAITING_TIME) - if not aoi_map: - grass.run_command("g.region", raster=f"{raster_name}.1") - - # reproject data - res = float( - grass.parse_command("r.info", flags="g", map=f"{raster_name}.1")[ - "nsres" - ] - ) - # switch location - os.environ["GISRC"] = str(gisrc) - if aoi_map: - grass.run_command("g.region", vector=aoi_map, res=res, flags="a") - else: - grass.run_command("g.region", res=res, flags="a") - for i in range(1, 5): - name = f"{raster_name}.{i}" - # set memory manually to 1000 - # Process stuck, when memory is too large (100000) - # GDAL_CACHEMAX is only interpreted as MB, if value is <100000 - grass.run_command( - "r.proj", - location=tmp_loc, - mapset="PERMANENT", - input=name, - output=name, - resolution=res, - flags="n", - quiet=True, - memory=1000, - ) - -def main(): # parser options tile_key = options["tile_key"] tile_url = options["tile_url"] @@ -285,6 +151,7 @@ def main(): orig_region = options["orig_region"] new_mapset = options["new_mapset"] download_dir = options["download_dir"] + keep_data = flags["k"] # set memory to input if possible options["memory"] = test_memory(options["memory"]) @@ -305,14 +172,17 @@ def main(): ) # import and reproject DOP tiles based on tileindex - import_and_reproject( + gisdbase, tmp_loc, tmp_gisrc = import_and_reproject( tile_url, raster_name, resolution_to_import, + "NW", aoi_map, download_dir, epsg=25832, + keep_data=keep_data, ) + # adjust resolution if required for band in [1, 2, 3, 4]: raster_name_band = f"{raster_name}.{band}" diff --git a/r.dop.import.worker.sn/r.dop.import.worker.sn.py b/r.dop.import.worker.sn/r.dop.import.worker.sn.py index 8e4d2b9..9049e73 100644 --- a/r.dop.import.worker.sn/r.dop.import.worker.sn.py +++ b/r.dop.import.worker.sn/r.dop.import.worker.sn.py @@ -5,8 +5,10 @@ # MODULE: r.dop.import.worker.sn # AUTHOR(S): Johannes Halbauer & Lina Krisztian # -# PURPOSE: Downloads Digital Orthophotos (DOPs) within a specified area in Sachsen -# COPYRIGHT: (C) 2024 by mundialis GmbH & Co. KG and the GRASS Development Team +# PURPOSE: Downloads Digital Orthophotos (DOPs) within a specified area +# in Sachsen +# COPYRIGHT: (C) 2024 by mundialis GmbH & Co. KG and the GRASS Development +# Team # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -99,25 +101,15 @@ import atexit -import os import sys -from time import sleep import grass.script as grass from grass.pygrass.utils import get_lib_path from grass_gis_helpers.cleanup import general_cleanup, cleaning_tmp_location from grass_gis_helpers.general import test_memory -from grass_gis_helpers.location import ( - get_current_location, - create_tmp_location, - switch_back_original_location, -) +from grass_gis_helpers.location import switch_back_original_location from grass_gis_helpers.mapset import switch_to_new_mapset -from grass_gis_helpers.open_geodata_germany.download_data import ( - download_data_using_threadpool, - extract_compressed_files, -) from grass_gis_helpers.raster import adjust_raster_resolution # import module library @@ -126,7 +118,7 @@ grass.fatal("Unable to find the dop library directory.") sys.path.append(path) try: - from r_dop_import_lib import rescale_to_1_256 + from r_dop_import_lib import rescale_to_1_256, import_and_reproject except Exception as imp_err: grass.fatal(f"r.dop.import library could not be imported: {imp_err}") @@ -137,9 +129,6 @@ tmp_loc = None tmp_gisrc = None -RETRIES = 30 -WAITING_TIME = 10 - def cleanup(): cleaning_tmp_location( @@ -151,133 +140,9 @@ def cleanup(): ) -def import_and_reproject( - url, - raster_name, - resolution_to_import, - aoi_map=None, - download_dir=None, - epsg=None, -): - """Import DOPs and reproject them if needed. - - Args: - url (str): The URL of the DOP to import - raster_name (str): The prefix name for the output rasters - aoi_map (str): Name of AOI vector map - download_dir (str): Path to local directory to downlaod DOPs to - epsg (int): EPSG code which has to be set if the reproduction should be - done manually and not by r.import - """ +def main(): global gisdbase, tmp_loc, tmp_gisrc - aoi_map_to_set_region1 = aoi_map - - # get actual location, mapset, ... - loc, mapset, gisdbase, gisrc = get_current_location() - if not aoi_map: - aoi_map = f"region_aoi_{grass.tempname(8)}" - aoi_map_to_set_region1 = aoi_map - grass.run_command("v.in.region", output=aoi_map, quiet=True) - else: - aoi_map_mapset = aoi_map.split("@") - aoi_map_to_set_region1 = aoi_map_mapset[0] - if len(aoi_map_mapset) == 2: - mapset = aoi_map_mapset[1] - # create temporary location with EPSG:25832 - tmp_loc, tmp_gisrc = create_tmp_location(epsg) - - # reproject aoi - if aoi_map: - grass.run_command( - "v.proj", - location=loc, - mapset=mapset, - input=aoi_map_to_set_region1, - output=aoi_map_to_set_region1, - quiet=True, - ) - grass.run_command( - "g.region", - vector=aoi_map_to_set_region1, - res=resolution_to_import, - flags="a", - ) - - # import data - # set memory manually to 1000 - # Process stuck, when memory is too large (100000) - # GDAL_CACHEMAX wird nur als MB interpretiert - kwargs = { - "input": url, - "output": raster_name, - "memory": 1000, - "quiet": True, - "extent": "region", - } - # TODO: Auslagern dieser Funktion in Lib, jedoch unterscheidet - # sich der folgende if-Block bspw. von dem von BB/BE - # - # download and keep data to download dir if -k flag ist set - # and change input parameter in kwargs to local path - if flags["k"]: - basename = os.path.basename(url) - print(basename) - url = os.path.dirname(url.replace("/vsizip/vsicurl/", "")) - download_data_using_threadpool([url], download_dir, None) - extract_compressed_files([os.path.basename(url)], download_dir) - kwargs["input"] = os.path.join(download_dir, basename) - - import_sucess = False - tries = 0 - while not import_sucess: - tries += 1 - if tries > RETRIES: - grass.fatal( - _( - f"Importing {kwargs['input']} failed after {RETRIES} " - "retries." - ) - ) - try: - grass.run_command("r.import", **kwargs) - import_sucess = True - except Exception: - sleep(WAITING_TIME) - if not aoi_map: - grass.run_command("g.region", raster=f"{raster_name}.1") - - # reproject data - res = float( - grass.parse_command("r.info", flags="g", map=f"{raster_name}.1")[ - "nsres" - ] - ) - # switch location - os.environ["GISRC"] = str(gisrc) - if aoi_map: - grass.run_command("g.region", vector=aoi_map, res=res, flags="a") - else: - grass.run_command("g.region", res=res, flags="a") - for i in range(1, 5): - name = f"{raster_name}.{i}" - # set memory manually to 1000 - # Process stuck, when memory is too large (100000) - # GDAL_CACHEMAX is only interpreted as MB, if value is <100000 - grass.run_command( - "r.proj", - location=tmp_loc, - mapset="PERMANENT", - input=name, - output=name, - resolution=res, - flags="n", - quiet=True, - memory=1000, - ) - - -def main(): # parser options tile_key = options["tile_key"] tile_url = options["tile_url"] @@ -286,6 +151,7 @@ def main(): orig_region = options["orig_region"] new_mapset = options["new_mapset"] download_dir = options["download_dir"] + keep_data = flags["k"] # set memory to input if possible options["memory"] = test_memory(options["memory"]) @@ -306,13 +172,15 @@ def main(): ) # import and reproject DOP tiles based on tileindex - import_and_reproject( + gisdbase, tmp_loc, tmp_gisrc = import_and_reproject( tile_url, raster_name, resolution_to_import, + "SN", aoi_map, download_dir, epsg=25832, + keep_data=keep_data, ) # adjust resolution if required for band in [1, 2, 3, 4]: diff --git a/r.dop.import.worker.th/r.dop.import.worker.th.py b/r.dop.import.worker.th/r.dop.import.worker.th.py index 77726c0..7a93895 100644 --- a/r.dop.import.worker.th/r.dop.import.worker.th.py +++ b/r.dop.import.worker.th/r.dop.import.worker.th.py @@ -5,8 +5,10 @@ # MODULE: r.dop.import.worker.th # AUTHOR(S): Johannes Halbauer & Lina Krisztian # -# PURPOSE: Downloads Digital Orthophotos (DOPs) within a specified area in Thüringen -# COPYRIGHT: (C) 2024 by mundialis GmbH & Co. KG and the GRASS Development Team +# PURPOSE: Downloads Digital Orthophotos (DOPs) within a specified area +# in Thüringen +# COPYRIGHT: (C) 2024 by mundialis GmbH & Co. KG and the GRASS Development +# Team # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/r.dop.import/r.dop.import.py b/r.dop.import/r.dop.import.py index f47ef90..0f04169 100644 --- a/r.dop.import/r.dop.import.py +++ b/r.dop.import/r.dop.import.py @@ -142,24 +142,6 @@ def cleanup(): ) -def run_old_addon(aoi, output, nativ_res): - """Run old, not refactored addon""" - r_dop_old_flags = "" - if nativ_res: - r_dop_old_flags += "r" - grass.run_command( - "r.dop.import.old", - aoi_map=aoi, - federal_state=options["federal_state"], - filepath=options["federal_state_file"], - local_data_dir=options["local_data_dir"], - nprocs=options["nprocs"], - flags=r_dop_old_flags, - output=output, - quiet=True, - ) - - def import_local_data(aoi, out, local_data_dir, fs, all_dops, native_res_flag): """Import local DOP data @@ -242,13 +224,9 @@ def main(): if not imported_local_data: # implement data download and import from open data out_fs = f"dop_{fs}_{ID}" - # run old addons - # TODO: modify list in if condition when adding a new refactored addon - if fs in NOT_YET_SUPPORTED and fs in ["SN", "TH", "HE"]: - grass.message(_("Using r.dop.import.old addon...")) - run_old_addon(aoi, out_fs, native_res) + # not yet supported - elif fs in NOT_YET_SUPPORTED: + if fs in NOT_YET_SUPPORTED: grass.fatal( _( "The import of the open data is not yet supported for " diff --git a/testsuite/test_r_dop_import_SN.py b/testsuite/test_r_dop_import_SN.py index cf1e668..031a84b 100644 --- a/testsuite/test_r_dop_import_SN.py +++ b/testsuite/test_r_dop_import_SN.py @@ -28,7 +28,7 @@ class TestRDopImportSN(RDopImportTestBase): fs = "SN" ref_res = 0.2 - aoi_cells = 2027082 + aoi_cells = 2021300 def test_default_settings(self): """