From 037a5ed0fc11bfd1f267cd1c3309ab9c0049fa7e Mon Sep 17 00:00:00 2001 From: Anna Petrasova Date: Tue, 20 Aug 2024 18:39:18 -0400 Subject: [PATCH] g.download.project: rename g.download.location (#4187) --- .github/workflows/macos.yml | 2 +- .github/workflows/test_thorough.bat | 2 +- .github/workflows/test_thorough.sh | 2 +- scripts/Makefile | 1 + .../g.download.location.html | 43 +---- .../g.download.location.py | 115 +------------ scripts/g.download.project/Makefile | 7 + .../g.download.project.html | 46 ++++++ .../g.download.project/g.download.project.py | 153 ++++++++++++++++++ 9 files changed, 224 insertions(+), 147 deletions(-) create mode 100644 scripts/g.download.project/Makefile create mode 100644 scripts/g.download.project/g.download.project.html create mode 100644 scripts/g.download.project/g.download.project.py diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index bf3859334ec..5a82b1020ea 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -88,7 +88,7 @@ jobs: shell: bash -el {0} run: | grass --tmp-project XY --exec \ - g.download.location url=${{ env.SampleData }} path=$HOME + g.download.project url=${{ env.SampleData }} path=$HOME grass --tmp-project XY --exec \ python3 -m grass.gunittest.main \ --grassdata $HOME --location nc_spm_full_v2alpha2 --location-type nc \ diff --git a/.github/workflows/test_thorough.bat b/.github/workflows/test_thorough.bat index 4b03a5608be..963f24b9b43 100644 --- a/.github/workflows/test_thorough.bat +++ b/.github/workflows/test_thorough.bat @@ -1,5 +1,5 @@ set grass=%1 set python=%2 -call %grass% --tmp-project XY --exec g.download.location url=https://grass.osgeo.org/sampledata/north_carolina/nc_spm_full_v2alpha2.tar.gz path=%USERPROFILE% +call %grass% --tmp-project XY --exec g.download.project url=https://grass.osgeo.org/sampledata/north_carolina/nc_spm_full_v2alpha2.tar.gz path=%USERPROFILE% call %grass% --tmp-project XY --exec %python% -m grass.gunittest.main --grassdata %USERPROFILE% --location nc_spm_full_v2alpha2 --location-type nc --min-success 80 diff --git a/.github/workflows/test_thorough.sh b/.github/workflows/test_thorough.sh index dba6b94b77d..6ed7d22078a 100755 --- a/.github/workflows/test_thorough.sh +++ b/.github/workflows/test_thorough.sh @@ -4,7 +4,7 @@ set -e grass --tmp-project XY --exec \ - g.download.location url=https://grass.osgeo.org/sampledata/north_carolina/nc_spm_full_v2alpha2.tar.gz path=$HOME + g.download.project url=https://grass.osgeo.org/sampledata/north_carolina/nc_spm_full_v2alpha2.tar.gz path=$HOME grass --tmp-project XY --exec \ python3 -m grass.gunittest.main \ diff --git a/scripts/Makefile b/scripts/Makefile index 84a9ad21e26..dc3d80f4209 100644 --- a/scripts/Makefile +++ b/scripts/Makefile @@ -19,6 +19,7 @@ SUBDIRS = \ db.test \ db.univar \ g.download.location \ + g.download.project \ g.extension \ g.extension.all \ g.manual \ diff --git a/scripts/g.download.location/g.download.location.html b/scripts/g.download.location/g.download.location.html index 6900acc4388..d9a73dc5513 100644 --- a/scripts/g.download.location/g.download.location.html +++ b/scripts/g.download.location/g.download.location.html @@ -1,47 +1,16 @@

DESCRIPTION

-g.download.location downloads an archived (e.g., -.zip or .tar.gz) project (previously called -location) from a given URL -and unpacks it to a specified or current GRASS GIS Spatial Database. -URL can be also a local file on the disk. - -If the archive contains a directory which contains a project, the module -will recognize that and use the project automatically. -The first directory which is a project is used. -Other projects or any other files are ignored. - -

EXAMPLES

- -

Download the full GRASS GIS sample project within a running session

- -Download and unpack the full North Carolina sample project into the user's -HOME directory: - -
-g.download.location url=https://grass.osgeo.org/sampledata/north_carolina/nc_spm_full_v2alpha2.tar.gz path=$HOME
-
- -

Download the full GRASS GIS sample project in a temporary session

- -In a temporary session, download and unpack the full North Carolina sample project -into the user's HOME directory: - -
-grass --tmp-location XY --exec g.download.location url=https://grass.osgeo.org/sampledata/north_carolina/nc_spm_full_v2alpha2.tar.gz path=$HOME
-
- +g.download.location has been renamed to +g.download.project +and exists for backwards compatibility reasons. +It will be removed in the next major version.

SEE ALSO

- g.mapset, - g.mapsets, - r.proj, - v.proj, - g.proj.all + g.download.project

AUTHOR

-Vaclav Petras, NCSU GeoForAll Lab +Vaclav Petras, NCSU GeoForAll Lab diff --git a/scripts/g.download.location/g.download.location.py b/scripts/g.download.location/g.download.location.py index bca79b12492..0e58a1ba395 100644 --- a/scripts/g.download.location/g.download.location.py +++ b/scripts/g.download.location/g.download.location.py @@ -3,8 +3,8 @@ # # MODULE: g.download.location # AUTHOR(S): Vaclav Petras -# PURPOSE: Download and extract location from web -# COPYRIGHT: (C) 2017 by the GRASS Development Team +# PURPOSE: Download and extract project (location) from web +# COPYRIGHT: (C) 2017-2024 by the GRASS Development Team # # This program is free software under the GNU General # Public License (>=v2). Read the file COPYING that @@ -12,11 +12,11 @@ # ############################################################################# -"""Download GRASS Locations""" +"""Download GRASS projects""" # %module -# % label: Download GRASS Location from the web -# % description: Get GRASS Location from an URL or file path +# % label: Download GRASS project (location) from the web +# % description: Get GRASS project from an URL or file path # % keyword: general # % keyword: data # % keyword: download @@ -26,7 +26,7 @@ # % key: url # % multiple: no # % type: string -# % label: URL of the archive with a location to be downloaded +# % label: URL of the archive with a project to be downloaded # % description: URL of ZIP, TAR.GZ, or other similar archive # % required: yes # %end @@ -42,111 +42,12 @@ # % multiple: no # %end -import atexit -import os -import shutil -from pathlib import Path - import grass.script as gs -from grass.grassdb.checks import is_location_valid -from grass.script.utils import try_rmdir -from grass.utils.download import DownloadError, download_and_extract, name_from_url - - -def find_location_in_directory(path, recurse=0): - """Return path to location in one of the subdirectories or None - - The first location found is returned. The expected usage is looking for one - location somewhere nested in subdirectories. - - By default only the immediate subdirectories of the provided directory are - tested, but with ``recurse >= 1`` additional levels of subdirectories - are tested for being locations. - - Directory names are sorted to provide a stable result. - - :param path: Path to the directory to search - :param recurse: How many additional levels of subdirectories to explore - """ - assert recurse >= 0 - full_paths = [os.path.join(path, i) for i in os.listdir(path)] - candidates = sorted([i for i in full_paths if os.path.isdir(i)]) - for candidate in candidates: - if is_location_valid(candidate): - return candidate - if recurse: - for candidate in candidates: - result = find_location_in_directory(candidate, recurse - 1) - if result: - return result - return None - - -def location_name_from_url(url): - """Create location name from URL""" - return gs.legalize_vector_name(name_from_url(url)) def main(options, unused_flags): - """Download and copy location to destination""" - url = options["url"] - name = options["name"] - database = options["path"] - - if not database: - # Use the current database path. - database = gs.gisenv()["GISDBASE"] - if not name: - name = location_name_from_url(url) - destination = Path(database) / name - - if destination.exists(): - gs.fatal( - _( - "Location named <{name}> already exists in <{directory}>, download canceled" - ).format(name=name, directory=database) - ) - - gs.message(_("Downloading and extracting...")) - try: - directory = download_and_extract(url) - if not directory.is_dir(): - gs.fatal(_("Archive contains only one file and no mapset directories")) - atexit.register(lambda: try_rmdir(directory)) - except DownloadError as error: - gs.fatal(_("Unable to get the location: {error}").format(error=error)) - if not is_location_valid(directory): - gs.verbose(_("Searching for valid location...")) - # This in fact deal with location being on the third level of directories - # thanks to how the extraction functions work (leaving out one level). - result = find_location_in_directory(directory, recurse=1) - if result: - # We just want to show relative path in the message. - # The relative path misses the root directory (name), because we - # loose it on the way. (We should use parent directory to get the - # full relative path, but the directory name is different now. - # This is the consequence of how the extract functions work.) - relative = os.path.relpath(result, start=directory) - gs.verbose( - _("Location found in a nested directory '{directory}'").format( - directory=relative - ) - ) - directory = result - else: - # The list is similarly misleading as the relative path above - # as it misses the root directory, but it still should be useful. - files_and_dirs = os.listdir(directory) - gs.fatal( - _( - "The downloaded file is not a valid GRASS Location." - " The extracted file contains these files and directories:" - "\n{files_and_dirs}" - ).format(files_and_dirs=" ".join(files_and_dirs)) - ) - gs.verbose(_("Copying to final destination...")) - shutil.copytree(src=directory, dst=destination) - gs.message(_("Path to the location now <{path}>").format(path=destination)) + """Download and copy project to destination""" + gs.run_command("g.dowload.project", **options) if __name__ == "__main__": diff --git a/scripts/g.download.project/Makefile b/scripts/g.download.project/Makefile new file mode 100644 index 00000000000..0f5e9d1c504 --- /dev/null +++ b/scripts/g.download.project/Makefile @@ -0,0 +1,7 @@ +MODULE_TOPDIR = ../.. + +PGM = g.download.project + +include $(MODULE_TOPDIR)/include/Make/Script.make + +default: script diff --git a/scripts/g.download.project/g.download.project.html b/scripts/g.download.project/g.download.project.html new file mode 100644 index 00000000000..7fc469fb0a6 --- /dev/null +++ b/scripts/g.download.project/g.download.project.html @@ -0,0 +1,46 @@ +

DESCRIPTION

+ +g.download.project downloads an archived (e.g., +.zip or .tar.gz) project (previously called +location) from a given URL +and unpacks it to a specified or current GRASS GIS Spatial Database. +URL can be also a local file on the disk. + +If the archive contains a directory which contains a project, the module +will recognize that and use the project automatically. +The first directory which is a project is used. +Other projects or any other files are ignored. + +

EXAMPLES

+ +

Download the full GRASS GIS sample project within a running session

+ +Download and unpack the full North Carolina sample project into the user's +HOME directory: + +
+g.download.project url=https://grass.osgeo.org/sampledata/north_carolina/nc_spm_full_v2alpha2.tar.gz path=$HOME
+
+ +

Download the full GRASS GIS sample project in a temporary session

+ +In a temporary session, download and unpack the full North Carolina sample project +into the user's HOME directory: + +
+grass --tmp-project XY --exec g.download.project url=https://grass.osgeo.org/sampledata/north_carolina/nc_spm_full_v2alpha2.tar.gz path=$HOME
+
+ +

SEE ALSO

+ + + g.mapset, + g.mapsets, + r.proj, + v.proj, + g.proj.all + + +

AUTHOR

+ +Vaclav Petras, NCSU GeoForAll Lab diff --git a/scripts/g.download.project/g.download.project.py b/scripts/g.download.project/g.download.project.py new file mode 100644 index 00000000000..aa963c31dd3 --- /dev/null +++ b/scripts/g.download.project/g.download.project.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +############################################################################ +# +# MODULE: g.download.project +# AUTHOR(S): Vaclav Petras +# PURPOSE: Download and extract project (location) from web +# COPYRIGHT: (C) 2017-2024 by the GRASS Development Team +# +# This program is free software under the GNU General +# Public License (>=v2). Read the file COPYING that +# comes with GRASS for details. +# +############################################################################# + +"""Download GRASS projects""" + +# %module +# % label: Download GRASS project from the web +# % description: Get GRASS project from an URL or file path +# % keyword: general +# % keyword: data +# % keyword: download +# % keyword: import +# %end +# %option +# % key: url +# % multiple: no +# % type: string +# % label: URL of the archive with a project to be downloaded +# % description: URL of ZIP, TAR.GZ, or other similar archive +# % required: yes +# %end +# %option G_OPT_M_LOCATION +# % key: name +# % required: no +# % multiple: no +# % key_desc: name +# %end +# %option G_OPT_M_DBASE +# % key: path +# % required: no +# % multiple: no +# %end + +import atexit +import os +import shutil +from pathlib import Path + +import grass.script as gs +from grass.grassdb.checks import is_location_valid +from grass.script.utils import try_rmdir +from grass.utils.download import DownloadError, download_and_extract, name_from_url + + +def find_location_in_directory(path, recurse=0): + """Return path to location in one of the subdirectories or None + + The first location found is returned. The expected usage is looking for one + location somewhere nested in subdirectories. + + By default only the immediate subdirectories of the provided directory are + tested, but with ``recurse >= 1`` additional levels of subdirectories + are tested for being locations. + + Directory names are sorted to provide a stable result. + + :param path: Path to the directory to search + :param recurse: How many additional levels of subdirectories to explore + """ + assert recurse >= 0 + full_paths = [os.path.join(path, i) for i in os.listdir(path)] + candidates = sorted([i for i in full_paths if os.path.isdir(i)]) + for candidate in candidates: + if is_location_valid(candidate): + return candidate + if recurse: + for candidate in candidates: + result = find_location_in_directory(candidate, recurse - 1) + if result: + return result + return None + + +def location_name_from_url(url): + """Create location name from URL""" + return gs.legalize_vector_name(name_from_url(url)) + + +def main(options, unused_flags): + """Download and copy location to destination""" + url = options["url"] + name = options["name"] + database = options["path"] + + if not database: + # Use the current database path. + database = gs.gisenv()["GISDBASE"] + if not name: + name = location_name_from_url(url) + destination = Path(database) / name + + if destination.exists(): + gs.fatal( + _( + "Project named <{}> already exists in <{directory}>, download canceled" + ).format(name=name, directory=database) + ) + + gs.message(_("Downloading and extracting...")) + try: + directory = download_and_extract(url) + if not directory.is_dir(): + gs.fatal(_("Archive contains only one file and no mapset directories")) + atexit.register(lambda: try_rmdir(directory)) + except DownloadError as error: + gs.fatal(_("Unable to get the project: {error}").format(error=error)) + if not is_location_valid(directory): + gs.verbose(_("Searching for valid project...")) + # This in fact deal with location being on the third level of directories + # thanks to how the extraction functions work (leaving out one level). + result = find_location_in_directory(directory, recurse=1) + if result: + # We just want to show relative path in the message. + # The relative path misses the root directory (name), because we + # loose it on the way. (We should use parent directory to get the + # full relative path, but the directory name is different now. + # This is the consequence of how the extract functions work.) + relative = os.path.relpath(result, start=directory) + gs.verbose( + _("Project found in a nested directory '{directory}'").format( + directory=relative + ) + ) + directory = result + else: + # The list is similarly misleading as the relative path above + # as it misses the root directory, but it still should be useful. + files_and_dirs = os.listdir(directory) + gs.fatal( + _( + "The downloaded file is not a valid GRASS project." + " The extracted file contains these files and directories:" + "\n{files_and_dirs}" + ).format(files_and_dirs=" ".join(files_and_dirs)) + ) + gs.verbose(_("Copying to final destination...")) + shutil.copytree(src=directory, dst=destination) + gs.message(_("Path to the project now <{path}>").format(path=destination)) + + +if __name__ == "__main__": + main(*gs.parser())