Merge pull request NRCan#566 from valhassan/565-feature-geo-inference…

…-integration

565 feature geo inference integration

.github/workflows/github-actions-ci.yml

@@ -1,60 +1,22 @@
-name: GitHub Actions Unit Tests
+name: tests
 on: [push, pull_request]
 jobs:
-  clean-and-deploy-env:
-    runs-on: ubuntu-latest
-    env:
-      LD_LIBRARY_PATH: "/opt/conda/lib"
+  run-tests:
+    runs-on: ubuntu-22.04
+    defaults:
+      run:
+        shell: bash -el {0}
     steps:
-      - name: Free Disk Space (Ubuntu)
-        uses: jlumbroso/free-disk-space@main
+      - name: checkout repository
+        uses: actions/[email protected]
+
+      - name: create environment
+        uses: conda-incubator/setup-miniconda@v3
         with:
-          # this might remove tools that are actually needed,
-          # if set to "true" but frees about 6 GB
-          tool-cache: false
-
-          # all of these default to true, but feel free to set to
-          # "false" if necessary for your workflow
-          android: true
-          dotnet: false
-          haskell: false
-          large-packages: false
-          docker-images: false
-          swap-storage: false
-
-      - name: Update the environment
+          mamba-version: "*"
+          activate-environment: geo_deep_env
+          environment-file: environment.yml
+
+      - name: test with pytest
         run: |
-          sudo apt-get update \
-          && sudo apt-get install -y --no-install-recommends git wget unzip bzip2 build-essential
-
-      - name: Install Mamba
-        run: |
-          sudo wget https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-Linux-x86_64.sh -O /tmp/mamba.sh && \
-          /bin/bash /tmp/mamba.sh -b -p /opt/conda && \
-          sudo rm -rf /tmp/* && \
-          sudo apt-get clean && \
-          sudo rm -rf /var/lib/apt/lists/*
-
-      - name: Set system variables
-        run: |
-          echo '/opt/conda' >> $GITHUB_PATH
-
-      - name: Check if Custom Paths are Added
-        run: |
-          echo "$PATH"
-          echo "$LD_LIBRARY_PATH"
-
-      - name: Install pyenv
-        run: |
-          pip install pipenv
-          pip install pytest
-      - uses: actions/checkout@v2
-
-      - name: Install dependencies
-        run: |
-          /opt/conda/bin/mamba env create -f environment.yml
-          echo '/opt/conda/envs/geo_deep_env/bin' >> $GITHUB_PATH
-
-      - name: Test with pytest
-        run: |
-          pipenv run pytest tests/
+          pytest tests/

config/augmentation/basic_augmentation_segmentation.yaml

@@ -11,7 +11,7 @@ augmentation:
 
   # Augmentations done immediately before conversion to torch tensor
   normalization: # Normalization: parameters for finetuning. See examples below:
-    mean:  # -> mean: [0.485, 0.456, 0.406]
-    std:  # -> std: std: [0.229, 0.224, 0.225])
+    mean: [0.0, 0.0, 0.0] # -> mean: [0.485, 0.456, 0.406]
+    std: [1.0, 1.0, 1.0] # -> std: std: [0.229, 0.224, 0.225])
   scale_data: [ 0, 1 ]  # Min and max value to scale values of input imagery
 

config/inference/default_binary.yaml

@@ -3,13 +3,13 @@ inference:
   raw_data_csv: tests/inference/inference_segmentation_binary.csv
   root_dir: inferences
   input_stac_item:  # alternatively, use a path or url to stac item directly
-  state_dict_path: ${general.save_weights_dir}/
+  model_path: ${general.save_weights_dir}/
   output_path:
   checkpoint_dir:  # (string, optional): directory in which to save the object if url
+  batch_size: 8
   chunk_size:  # if empty, will be calculated automatically from max_pix_per_mb_gpu
   # Maximum number of pixels each Mb of GPU Ram to allow. E.g. if GPU has 1000 Mb of Ram and this parameter is set to
   # 10, chunk_size will be set to sqrt(1000 * 10) = 100.
-  use_hanning: True # enables smoothening with Hann windows; creates overlapping tiles.
   max_pix_per_mb_gpu: 25
   prep_data_only: False
   override_model_params: False
@@ -22,4 +22,4 @@ inference:
   max_used_ram: ${training.max_used_ram}  # If RAM usage of detected GPU exceeds this percentage, it will be ignored
 
   # Post-processing
-  ras2vec: False  # if True, a polygonized version of the inference (.gpkg) will be created with rasterio tools
+  mask_to_vector: False  # if True, a polygonized version of the inference (.gpkg) will be created with rasterio tools

config/inference/default_multiclass.yaml

@@ -3,9 +3,10 @@ inference:
   raw_data_csv: tests/inference/inference_segmentation_multiclass.csv
   root_dir: inferences
   input_stac_item:  # alternatively, use a path or url to stac item directly
-  state_dict_path: ${general.save_weights_dir}/
+  model_path: ${general.save_weights_dir}/
   output_path:
   checkpoint_dir:  # (string, optional): directory in which to save the object if url
+  batch_size: 8
   chunk_size:  # if empty, will be calculated automatically from max_pix_per_mb_gpu
   # Maximum number of pixels each Mb of GPU Ram to allow. E.g. if GPU has 1000 Mb of Ram and this parameter is set to
   # 10, chunk_size will be set to sqrt(1000 * 10) = 100.
@@ -21,4 +22,4 @@ inference:
   max_used_ram: ${training.max_used_ram}  # If RAM usage of detected GPU exceeds this percentage, it will be ignored
 
   # Post-processing
-  ras2vec: False  # if True, a polygonized version of the inference (.gpkg) will be created with rasterio tools
+  mask_to_vector: False  # if True, a polygonized version of the inference (.gpkg) will be created with rasterio tools

config/training/default_training.yaml

@@ -13,7 +13,7 @@ training:
   max_used_perc:
   state_dict_path:
   state_dict_strict_load: True
-  script_model: False
+  script_model: True
   compute_sampler_weights: False
 
 #  precision: 16

environment.yml

@@ -1,26 +1,16 @@
 name: geo_deep_env
 channels:
-  - pytorch
   - conda-forge
 dependencies:
-  - coverage>=6.3.1
-  - docker-py>=4.4.4
-  - geopandas>=0.10.2
-  - hydra-core>=1.2.0
+  - python==3.11.5
   - pip
-  - pynvml>=11.0
+  - gdal
   - pystac>=0.3.0
   - pytest>=7.1
-  - python>=3.10
-  - pytorch==1.12
-  - mkl<=2024.0
-  - rich>=11.1
   - ruamel_yaml>=0.15
   - scikit-image>=0.18
-  - torchgeo>=0.3
-  - torchvision>=0.12
   - pip:
+    - geo-inference>=2.0.7
     - hydra-colorlog>=1.1.0
     - hydra-optuna-sweeper>=1.1.0
-    - ttach>=0.0.3
     - mlflow>=1.2 # causes env solving to hang if not with pip

evaluate_segmentation.py

@@ -124,7 +124,7 @@ def main(params):
     gpkg_name_ = []
 
     for aoi in tqdm(list_aois, desc='Evaluating from input list', position=0, leave=True):
-        output_path = working_folder / f"{aoi.aoi_id}_pred.tif" if not inference_image else inference_image
+        output_path = working_folder / f"{aoi.aoi_id}_mask.tif" if not inference_image else inference_image
         if not output_path.is_file():
             raise FileNotFoundError(f"Couldn't locate inference to evaluate metrics with. Make inference has been run "
                                     f"before you run evaluate mode.")