Merge pull request #6 from AgPipeline/develop

Merging develop into master - no review

Dockerfile

@@ -1,6 +1,54 @@
-FROM agdrone/agpypeline:1.0
+FROM ubuntu:18.04
 LABEL maintainer="Chris Schnaufer <[email protected]>"
 
+# Add user
+RUN useradd -u 49044 extractor \
+    && mkdir /home/extractor
+RUN chown -R extractor /home/extractor \
+    && chgrp -R extractor /home/extractor
+
+# Install the Python version we want
+RUN apt-get update && \
+    apt-get upgrade -y && \
+    apt-get install -y --no-install-recommends \
+        python3.7 \
+        python3-pip && \
+    ln -sfn /usr/bin/python3.7 /usr/bin/python && \
+    ln -sfn /usr/bin/python3.7 /usr/bin/python3 && \
+    ln -sfn /usr/bin/python3.7m /usr/bin/python3m && \
+    apt-get autoremove -y && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+# Perform some upgrades
+RUN python3 -m pip install --upgrade --no-cache-dir pip
+RUN python3 -m pip install --upgrade --no-cache-dir setuptools
+
+# Install applications we need
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+        python3-gdal \
+        gdal-bin   \
+        libgdal-dev  \
+        gcc \
+        g++ \
+        python3.7-dev && \
+    python3 -m pip install --upgrade --no-cache-dir \
+        wheel && \
+    python3 -m pip install --upgrade --no-cache-dir \
+        numpy && \
+    python3 -m pip install --upgrade --no-cache-dir \
+        pygdal==2.2.3.* && \
+    apt-get remove -y \
+        libgdal-dev \
+        gcc \
+        g++ \
+        python3-dev && \
+    apt-get autoremove -y && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+# Install the requirements we need for this Transformer
 COPY requirements.txt packages.txt /home/extractor/
 
 USER root

requirements.txt

@@ -1,2 +1,2 @@
 liblas
-agpypeline==0.0.20
+agpypeline

transformer.py

@@ -44,7 +44,7 @@
 }
 
 # Trait names arrays
-CSV_TRAIT_NAMES = ['germplasmName', 'site', 'timestamp', 'lat', 'lon', 'citation_author', 'citation_year', 'citation_title']
+CSV_TRAIT_NAMES = ['species', 'site', 'timestamp', 'lat', 'lon', 'citation_author', 'citation_year', 'citation_title']
 GEO_TRAIT_NAMES = ['site', 'trait', 'lat', 'lon', 'dp_time', 'source', 'value', 'timestamp']
 BETYDB_TRAIT_NAMES = ['local_datetime', 'access_level', 'species', 'site', 'citation_author', 'citation_year', 'citation_title',
                       'method']
@@ -517,13 +517,13 @@ def get_csv_traits_table(variable_names: list) -> tuple:
             traits[field_name] = __internal__.get_default_trait(field_name)
 
         if hasattr(algorithm_rgb, 'CITATION_AUTHOR') and getattr(algorithm_rgb, 'CITATION_AUTHOR'):
-            traits['citation_author'] = getattr(algorithm_rgb, 'CITATION_AUTHOR')
+            traits['citation_author'] = '"' + getattr(algorithm_rgb, 'CITATION_AUTHOR') + '"'
         if hasattr(algorithm_rgb, 'CITATION_TITLE') and getattr(algorithm_rgb, 'CITATION_TITLE'):
-            traits['citation_title'] = getattr(algorithm_rgb, 'CITATION_TITLE')
+            traits['citation_title'] = '"' + getattr(algorithm_rgb, 'CITATION_TITLE') + '"'
         if hasattr(algorithm_rgb, 'CITATION_YEAR') and getattr(algorithm_rgb, 'CITATION_YEAR'):
-            traits['citation_year'] = getattr(algorithm_rgb, 'CITATION_YEAR')
+            traits['citation_year'] = '"' + getattr(algorithm_rgb, 'CITATION_YEAR') + '"'
 
-        return (fields, traits)
+        return fields, traits
 
     @staticmethod
     def get_geo_traits_table() -> tuple:
@@ -536,7 +536,7 @@ def get_geo_traits_table() -> tuple:
         for field_name in fields:
             traits[field_name] = ""
 
-        return (fields, traits)
+        return fields, traits
 
     @staticmethod
     def get_bety_traits_table(variable_names: list) -> tuple:
@@ -553,15 +553,15 @@ def get_bety_traits_table(variable_names: list) -> tuple:
             traits[field_name] = __internal__.get_default_trait(field_name)
 
         if hasattr(algorithm_rgb, 'CITATION_AUTHOR') and getattr(algorithm_rgb, 'CITATION_AUTHOR'):
-            traits['citation_author'] = getattr(algorithm_rgb, 'CITATION_AUTHOR')
+            traits['citation_author'] = '"' + getattr(algorithm_rgb, 'CITATION_AUTHOR') + '"'
         if hasattr(algorithm_rgb, 'CITATION_TITLE') and getattr(algorithm_rgb, 'CITATION_TITLE'):
-            traits['citation_title'] = getattr(algorithm_rgb, 'CITATION_TITLE')
+            traits['citation_title'] = '"' + getattr(algorithm_rgb, 'CITATION_TITLE') + '"'
         if hasattr(algorithm_rgb, 'CITATION_YEAR') and getattr(algorithm_rgb, 'CITATION_YEAR'):
-            traits['citation_year'] = getattr(algorithm_rgb, 'CITATION_YEAR')
+            traits['citation_year'] = '"' + getattr(algorithm_rgb, 'CITATION_YEAR') + '"'
         if hasattr(algorithm_rgb, 'ALGORITHM_METHOD') and getattr(algorithm_rgb, 'ALGORITHM_METHOD'):
-            traits['method'] = getattr(algorithm_rgb, 'ALGORITHM_METHOD')
+            traits['method'] = '"' + getattr(algorithm_rgb, 'ALGORITHM_METHOD') + '"'
 
-        return (fields, traits)
+        return fields, traits
 
     @staticmethod
     def generate_traits_list(fields: list, traits: dict) -> list:
@@ -708,9 +708,9 @@ def add_parameters(self, parser: argparse.ArgumentParser) -> None:
         """
         # pylint: disable=no-self-use
         supported_files = [FILE_NAME_CSV + ': basic CSV file with calculated values']
-        if __internal__.get_algorithm_definition_bool('WRITE_GEOSTREAMS_CSV', True):
+        if __internal__.get_algorithm_definition_bool('WRITE_GEOSTREAMS_CSV', False):
             supported_files.append(FILE_NAME_BETYDB_CSV + ': TERRA REF Geostreams compatible CSV file')
-        if __internal__.get_algorithm_definition_bool('WRITE_BETYDB_CSV', True):
+        if __internal__.get_algorithm_definition_bool('WRITE_BETYDB_CSV', False):
             supported_files.append(FILE_NAME_BETYDB_CSV + ': BETYdb compatible CSV file')
 
         parser.description = 'Plot level RGB algorithm: ' + __internal__.get_algorithm_name() + \
@@ -782,7 +782,7 @@ def perform_process(self, environment: Environment, check_md: dict, transformer_
         logging.debug("Calculated geostreams CSV path: %s", geostreams_csv_file)
         logging.debug("Calculated BETYdb CSV path: %s", betydb_csv_file)
         datestamp, localtime = __internal__.get_time_stamps(check_md['timestamp'])
-        cultivar = __internal__.find_metadata_value(full_md, ['germplasmName', 'cultivar'])
+        species = __internal__.find_metadata_value(full_md, ['species', 'cultivar'])
 
         write_geostreams_csv = environment.args.geostreams_csv or __internal__.get_algorithm_definition_bool('WRITE_GEOSTREAMS_CSV', True)
         write_betydb_csv = environment.args.betydb_csv or __internal__.get_algorithm_definition_bool('WRITE_BETYDB_CSV', True)
@@ -791,10 +791,10 @@ def perform_process(self, environment: Environment, check_md: dict, transformer_
 
         # Get default values and adjust as needed
         (csv_fields, csv_traits) = __internal__.get_csv_traits_table(variable_names)
-        csv_traits['germplasmName'] = cultivar
+        csv_traits['species'] = species
         (geo_fields, geo_traits) = __internal__.get_geo_traits_table()
         (bety_fields, bety_traits) = __internal__.get_bety_traits_table(variable_names)
-        bety_traits['species'] = cultivar
+        bety_traits['species'] = species
 
         csv_header = ','.join(map(str, __internal__.get_csv_header_fields()))
         geo_csv_header = ','.join(map(str, geo_fields))
@@ -813,7 +813,7 @@ def perform_process(self, environment: Environment, check_md: dict, transformer_
                 # Setup
                 plot_name = __internal__.recursive_metadata_search(full_md, 'plot_name', one_file)
                 centroid = __internal__.get_centroid_latlon(one_file)
-                image_pix = np.array(gdal.Open(one_file).ReadAsArray())
+                image_pix = np.rollaxis(np.array(gdal.Open(one_file).ReadAsArray()), 0, 3)
 
                 # Make the call and check the results
                 calc_value = algorithm_rgb.calculate(image_pix)