Merge pull request #1 from mundialis/m.neural_network.preparedata

m.neural_network.preparedata: prepare data to create training data set

.flake8

@@ -0,0 +1,16 @@
+[flake8]
+
+# E402 module level import not at top of file
+# E501 line too long (83 > 79 characters)
+# F821 undefined name '_'
+
+exclude = .git
+
+max-line-length = 87
+
+per-file-ignores =
+    ./m.neural_network.preparedata/m.neural_network.preparedata.py: F821
+    ./m.neural_network.preparedata/m.neural_network.preparedata.py: E501
+    ./m.neural_network.preparedata.worker_nullcells/m.neural_network.preparedata.worker_nullcells.py: F821
+    ./m.neural_network.preparedata.worker_nullcells/m.neural_network.preparedata.worker_nullcells.py: E501
+    ./m.neural_network.preparedata.worker_export/m.neural_network.preparedata.worker_export.py: E501

.github/workflows/grass-manual.yml

@@ -0,0 +1,7 @@
+on:
+  push:
+    branches: [ main ]
+
+jobs:
+  grass-manual:
+    uses: mundialis/github-workflows/.github/workflows/grass-manual.yml@main

.github/workflows/linting.yml

@@ -0,0 +1,15 @@
+---
+name: Linting and code quality check
+
+on:
+  push:
+    branches:
+      - main
+      - develop
+  pull_request:
+
+jobs:
+  lint:
+    uses: mundialis/github-workflows/.github/workflows/linting.yml@main
+    with:
+      VALIDATE_HTML: false

.github/workflows/post-pr-reviews.yml

@@ -0,0 +1,12 @@
+---
+name: Post PR code suggestions
+
+on:
+  workflow_run:
+    workflows: ["Linting and code quality check"]
+    types:
+      - completed
+
+jobs:
+  post-pr-reviews:
+    uses: mundialis/github-workflows/.github/workflows/post-pr-reviews.yml@main

.pylintrc

@@ -0,0 +1,48 @@
+# For documentation about this config, see
+# https://pylint.readthedocs.io/en/stable/user_guide/configuration/all-options.html (as of writing, version 2.17.4)
+
+[MAIN]
+
+jobs=0 # Default: 1
+
+load-plugins=
+        pylint.extensions.broad_try_clause
+
+recursive=yes # Default: False
+
+# reports=yes # Default: False
+
+# score=no # Default: True
+
+# To disable more rules, see output of pylint. E.g.
+# [...] C0301: Line too long (89/80) (line-too-long)
+# can be suppressed with either disable=line-too-long or disable=C
+# It is also possible to ignore a specific line by adding
+# # pylint: disable=broad-exception-caught
+# above the line causing the lint error
+disable=
+        W, ; all Warnings are allowed to fail
+        import-error, ; To suppress e.g "Unable to import 'grass.script"
+        missing-module-docstring, ; we use the GRASS GIS header
+        R, ; refactoring + design recommendations
+        consider-using-enumerate, ; to supress consider using enumerate
+        line-too-long, ; >80
+
+[BASIC]
+# for global variable setting
+good-names=keep_data,download_dir,gisdbase,tgtgisrc,tmploc,srcgisrc
+
+[BROAD_TRY_CLAUSE]
+max-try-statements=4 # Default: 1
+
+[FORMAT]
+max-line-length=80 # Default: 100
+max-module-lines=800 # Default: 1000
+ignore-long-lines=.*COPYRIGHT:.* |# . description:.*|\s*(# )?.*http.:\/\/\S+?|# %%*
+
+[MESSAGES CONTROL]
+# E0606 (possibly-used-before-assignment): to set options and flags at the bottom of the file without pre-initialization
+disable=E0606
+
+[VARIABLES]
+additional-builtins=_ # Default: ()

.pylintrc_allowed_to_fail

@@ -0,0 +1,57 @@
+# For documentation about this config, see
+# https://pylint.readthedocs.io/en/stable/user_guide/configuration/all-options.html (as of writing, version 2.17.4)
+
+[MAIN]
+
+exit-zero=yes
+
+jobs=0 # Default: 1
+
+load-plugins=
+       pylint.extensions.broad_try_clause
+
+recursive=yes # Default: False
+
+# score=no # Default: True
+
+[BASIC]
+# for global variable setting
+good-names=keep_data,download_dir,gisdbase,tgtgisrc,tmploc,srcgisrc
+
+[BROAD_TRY_CLAUSE]
+max-try-statements=4 # Default: 1
+
+[FORMAT]
+max-line-length=80 # Default: 100
+max-module-lines=800 # Default: 1000
+ignore-long-lines=.*COPYRIGHT:.* |# . description:.*|\s*(# )?.*http.:\/\/\S+?|# %%*
+
+[MESSAGES CONTROL]
+# E0606 (possibly-used-before-assignment): to set options and flags at the bottom of the file without pre-initialization
+disable=E0606
+
+[VARIABLES]
+additional-builtins=_ # Default: ()
+
+
+; [DESIGN]
+
+; # Maximum number of arguments for function / method.
+; # Default: 5
+; max-args=9
+
+; # Maximum number of attributes for a class (see R0902).
+; # Default: 7
+; max-attributes=11
+
+; # Maximum number of branch for function / method body.
+; # Default: 12
+; max-branches=15
+
+; # Maximum number of locals for function / method body.
+; # Default: 15
+; max-locals=19
+
+; # Maximum number of return / yield for function / method body.
+; # Default: 6
+; max-returns=11

Makefile

@@ -0,0 +1,17 @@
+MODULE_TOPDIR = ../..
+
+PGM = m.neural_network
+
+# note: to deactivate a module, just place a file "DEPRECATED" into the subdir
+ALL_SUBDIRS := ${sort ${dir ${wildcard */.}}}
+DEPRECATED_SUBDIRS := ${sort ${dir ${wildcard */DEPRECATED}}}
+RM_SUBDIRS := bin/ docs/ etc/ scripts/
+SUBDIRS_1 := $(filter-out $(DEPRECATED_SUBDIRS), $(ALL_SUBDIRS))
+SUBDIRS := $(filter-out $(RM_SUBDIRS), $(SUBDIRS_1))
+
+include $(MODULE_TOPDIR)/include/Make/Dir.make
+
+default: parsubdirs htmldir
+
+install: installsubdirs
+	$(INSTALL_DATA) $(PGM).html $(INST_DIR)/docs/html/

README.md

@@ -0,0 +1,10 @@
+# m.neural_network - Toolset for creating training data and training a neural network
+
+For now the toolset only includes add-ons for data preparation for training data creation.
+
+The m.neural_network toolset consists of following modules:
+* m.neural_network.preparedata: prepare training data as first step for the process of creating a neural network.
+  * m.neural_network.preparedata.worker_nullsells: Worker module for m.neural_network.preparedata to check null cells
+  * m.neural_network.preparedata.worker_export: Worker module for m.neural_network.preparedata to export data
+* m.neural_network.preparetraining: prepare training data for use in model training
+  * m.neural_network.preparetraining.worker: Worker module for m.neural_network.preparetraining to check and rasterize label data

m.neural_network.html

@@ -0,0 +1,53 @@
+<!DOCTYPE html>
+<html>
+<head>
+<title>m.neural_network toolset - GRASS GIS manual</title>
+<meta name="Author" content="mundialis team">
+<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+<meta http-equiv="content-language" content="en-us">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<link rel="stylesheet" href="grassdocs.css" type="text/css">
+</head>
+<body bgcolor="white">
+<div id="container">
+
+<a href="https://grass.osgeo.org/grass-stable/manuals/index.html"><img src="grass_logo.png" alt="GRASS logo"></a>
+<hr class="header">
+
+<h2>NAME</h2>
+
+<em><b>m.neural_network</b></em> -  GRASS GIS addons to train and apply a neural network.
+
+<h2>KEYWORDS</h2>
+<a href="https://grass.osgeo.org/grass-stable/manuals/raster.html">raster</a>, <a href="https://grass.osgeo.org/grass-stable/manuals/vector.html">vector</a>
+
+<!-- meta page description:  addons for Regionalverband Ruhr (RVR) related geodata processing -->
+<h2>DESCRIPTION</h2>
+
+The <em>m.neural_network</em> toolset consists of several modules.
+
+<ul>
+<li><a href="m.neural_network.preparedata.html">m.neural_network.preparedata</a>: Prepares and exports tiles for the label process</li>
+<li><a href="m.neural_network.preparedata.worker_export.html">m.neural_network.preparedata.worker_export</a>: Worker for parallel processing for exporting for <b>m.neural_network.preparedata</b></li>
+<li><a href="m.neural_network.preparedata.worker_nullcells.html">m.neural_network.preparedata.worker_nullcells</a>: Worker to analyse the number of null cells in parallel for <b>m.neural_network.preparedata</b></li>
+<li><a href="m.neural_network.preparetraining.html">m.neural_network.preparetraining</a>: Takes and reformats labeled tiles for the neural network training</li>
+</ul>
+
+<h2>REQUIREMENTS</h2>
+
+The following Python libraries are needed.
+
+<ul>
+  <li>grass-gis-helpers>=2.2.0</li>
+</ul>
+
+<h2>AUTHORS</h2>
+
+Anika Weinmann, <a href="https://www.mundialis.de/">mundialis GmbH &amp; Co. KG</a>, weinmann at mundialis.de
+<p>Guido Riembauer, <a href="https://www.mundialis.de/">mundialis GmbH &amp; Co. KG</a>, riembauer at mundialis.de</p>
+<p>Victoria-Leandra Brunn, <a href="https://www.mundialis.de/">mundialis GmbH &amp; Co. KG</a>, brunn at mundialis.de</p>
+
+</div>
+</body>
+</html>
+

m.neural_network.preparedata.worker_export/Makefile

@@ -0,0 +1,7 @@
+MODULE_TOPDIR = ../..
+
+PGM = m.neural_network.preparedata.worker_export
+
+include $(MODULE_TOPDIR)/include/Make/Script.make
+
+default: script

m.neural_network.preparedata.worker_export/m.neural_network.preparedata.worker_export.html

@@ -0,0 +1,14 @@
+<h2>DESCRIPTION</h2>
+
+<em>m.neural_network.preparedata.worker_export</em> is used within <em>m.neural_network.preparedata</em> to export data in parallel.
+
+<h2>SEE ALSO</h2>
+
+<em>
+<a href="g.region.html">g.region</a>,
+<a href="r.univar.html">r.univar</a>
+</em>
+
+<h2>AUTHORS</h2>
+Anika Weinmann, <a href="https://www.mundialis.de/">mundialis GmbH &amp; Co. KG</a><br>
+<p>Guido Riembauer, <a href="https://www.mundialis.de/">mundialis GmbH &amp; Co. KG</a><br></p>