Migrate to QGIS 3 + Python 3

.travis.yml

@@ -3,7 +3,7 @@ sudo: required
 language: python
 
 python:
-  - "2.7"
+  - "3.6"
 
 env:
   global:
@@ -23,9 +23,6 @@ env:
       HEADLESS_COMMAND=dev
       SUBSEQUENT_RUN_TESTING=True
 
-virtualenv:
-  system_site_packages: true
-
 services:
   - docker
 

REQUIREMENTS-TRAVIS.txt

@@ -1,3 +1,3 @@
-ansible==2.3.1.0
+ansible>=2.6.18
 codecov
 flake8==3.6.0

deployment/ansible/development/group_vars/all.travis.yml

@@ -16,7 +16,7 @@ interpreters:
 inasafe:
   repo: https://github.com/inasafe/inasafe.git
   remote: upstream
-  version: inasafe_4
+  version: maintenance-fix
   depth: 1
 
 inasafe_headless_worker:

deployment/docker-headless/Dockerfile

@@ -1,5 +1,5 @@
 #--------- Generic stuff all our Dockerfiles should start with so we get caching ------------
-FROM kartoza/qgis-desktop:2.18
+FROM qgis/qgis:release-3_14
 
 RUN apt-get -y update; apt-get -y --force-yes install pwgen git inotify-tools
 
@@ -9,7 +9,10 @@ RUN apt-get -y update; apt-get -y --force-yes install pwgen git inotify-tools
 RUN apt-get update -y; apt-get install -y --force-yes openssh-server sudo
 RUN mkdir /var/run/sshd
 RUN echo 'root:docker' | chpasswd
-RUN sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config
+# Comment out PermitRootLogin setting, whatever it is
+RUN sed -i 's/^PermitRootLogin */#PermitRootLogin /' /etc/ssh/sshd_config
+# Write out PermitRootLogin setting at the end
+RUN echo "PermitRootLogin yes" >> /etc/ssh/sshd_config
 
 # SSH login fix. Otherwise user is kicked off after login
 RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd
@@ -21,8 +24,7 @@ RUN echo "export VISIBLE=now" >> /etc/profile
 
 #-------------Application Specific Stuff ----------------------------------------------------
 # Install git, xvfb
-RUN apt-get -y update; apt-get -y --force-yes install git xvfb python-setuptools python-dev libssl-dev libffi-dev python-scipy
-RUN easy_install pip==9.0.1
+RUN apt-get -y update; apt-get -y --force-yes install git xvfb python3-setuptools python3-dev libssl-dev libffi-dev python3-scipy
 # Copy ubuntu fonts
 RUN apt-get -y update; apt-get -y --force-yes install wget unzip
 ADD ubuntu-font-family-0.83.zip /ubuntu-font-family-0.83.zip
@@ -31,8 +33,9 @@ RUN mv ubuntu-font-family-0.83 /usr/share/fonts/truetype/ubuntu-font-family
 RUN fc-cache -f -v
 
 # This image instance uses dist-packages directory
-ADD sitecustomize.py /usr/local/lib/python2.7/dist-packages/sitecustomize.py
+ADD sitecustomize.py /usr/lib/python3/dist-packages/sitecustomize.py
 ADD REQUIREMENTS.txt /REQUIREMENTS.txt
+RUN ln -s /usr/bin/pip3 /usr/bin/pip
 RUN pip install -r REQUIREMENTS.txt
 
 ADD docker-entrypoint.sh /docker-entrypoint.sh

deployment/docker-headless/docker-entrypoint.sh

@@ -2,7 +2,7 @@
 
 # Wait run xvfb
 while [ -z  "$(pidof /usr/bin/Xvfb)" ]; do
-  start-stop-daemon --start -b -x /usr/bin/Xvfb ${DISPLAY}
+  start-stop-daemon --start -b -x /usr/bin/Xvfb ${DISPLAY} -- -screen 0 1024x768x24 -ac +extension GLX +render -noreset -nolisten tcp
   sleep 5
 done
 

deployment/production/docker/headless/Dockerfile

@@ -1,12 +1,11 @@
 #--------- Generic stuff all our Dockerfiles should start with so we get caching ------------
-FROM kartoza/qgis-desktop:2.18
+FROM qgis/qgis:release-3_14
 
 RUN apt-get -y update; apt-get -y --force-yes install pwgen git inotify-tools
 
 #-------------Application Specific Stuff ----------------------------------------------------
 # Install git, xvfb
-RUN apt-get -y update; apt-get -y --force-yes install git xvfb python-setuptools python-dev libssl-dev libffi-dev python-scipy
-RUN easy_install pip==9.0.1
+RUN apt-get -y update; apt-get -y --force-yes install git xvfb python3-setuptools python3-dev libssl-dev libffi-dev python3-scipy
 # Copy ubuntu fonts
 RUN apt-get -y update; apt-get -y --force-yes install wget unzip
 ADD ubuntu-font-family-0.83.zip /ubuntu-font-family-0.83.zip
@@ -15,6 +14,7 @@ RUN mv ubuntu-font-family-0.83 /usr/share/fonts/truetype/ubuntu-font-family
 RUN fc-cache -f -v
 
 ADD REQUIREMENTS.txt /REQUIREMENTS.txt
+RUN ln -s /usr/bin/pip3 /usr/bin/pip
 RUN pip install -r REQUIREMENTS.txt
 
 ADD docker-entrypoint.sh /docker-entrypoint.sh
@@ -23,7 +23,7 @@ RUN chmod +x /docker-entrypoint.sh
 # Install InaSAFE Core
 RUN mkdir -p /usr/share/qgis/python/plugins
 WORKDIR /usr/share/qgis/python/plugins
-ARG INASAFE_CORE_TAG=version-4_4_0
+ARG INASAFE_CORE_TAG=develop
 RUN git clone --branch ${INASAFE_CORE_TAG} --depth 1 --recursive https://github.com/inasafe/inasafe.git inasafe
 
 # Install InaSAFE Headless

deployment/production/docker/headless/docker-entrypoint.sh

@@ -2,11 +2,12 @@
 
 # Wait run xvfb
 while [ -z  "$(pidof /usr/bin/Xvfb)" ]; do
-  start-stop-daemon --start -b -x /usr/bin/Xvfb ${DISPLAY}
+  start-stop-daemon --start -b -x /usr/bin/Xvfb ${DISPLAY} -- -screen 0 1024x768x24 -ac +extension GLX +render -noreset -nolisten tcp
   sleep 5
 done
 
 cp -n /home/app/headless/celeryconfig_sample.py /home/app/headless/celeryconfig.py
+echo "Config file copied"
 
 if [ $# -eq 2 ] && [ $1 = "prod" ] && [ $2 = "inasafe-headless-worker" ]; then
     /usr/local/bin/celery -A headless.celery_app worker -l info -Q inasafe-headless -n inasafe-headless.%h

src/headless/REQUIREMENTS.txt

@@ -1,8 +1,11 @@
 requests>=2.6.2
 tzlocal==1.2
-celery==4.1.0
-nosexcover
+hammock==0.2.4
 nose2==0.6.5
 coverage==4.4.2
+threadpool==1.3.2
+pyinotify==0.9.6
+celery==4.1.1
 raven==5.29.0
 mock==2.0.0
+nose==1.3.7

src/headless/celery_app.py

@@ -1,5 +1,6 @@
 # coding=utf-8
 import importlib
+from importlib import reload
 import json
 import os
 
@@ -30,7 +31,7 @@ def load_inasafe_settings():
     from safe.definitions import default_settings
     from safe.utilities.settings import set_setting
 
-    for key, value in default_settings.inasafe_default_settings.iteritems():
+    for key, value in list(default_settings.inasafe_default_settings.items()):
         set_setting(key, value)
 
     # Override settings from INASAFE_SETTINGS_PATH

src/headless/settings.py

@@ -1,5 +1,6 @@
 # coding=utf-8
 """InaSAFE Headless settings."""
+from builtins import str
 import logging
 from distutils.util import strtobool
 

src/headless/tasks/inasafe_analysis.py

@@ -5,10 +5,10 @@
 
 from copy import deepcopy
 from datetime import datetime
-from PyQt4.QtCore import QUrl
+from qgis.PyQt.QtCore import QUrl
 
 from qgis.core import (
-    QgsCoordinateReferenceSystem, QgsMapLayerRegistry, QgsProject)
+    QgsCoordinateReferenceSystem, QgsProject)
 
 from safe.definitions.constants import (
     PREPARE_SUCCESS, ANALYSIS_SUCCESS, MULTI_EXPOSURE_ANALYSIS_FLAG)
@@ -55,7 +55,7 @@ def clean_metadata(metadata):
     :param metadata: Metadata as dictionary.
     :type metadata: dict
     """
-    for key, value in metadata.items():
+    for key, value in list(metadata.items()):
         if isinstance(value, dict):
             clean_metadata(value)
         if isinstance(value, QUrl):
@@ -118,7 +118,7 @@ def inasafe_analysis(
     """
     # Clean up layer registry before using
     # In case previous task exited prematurely before cleanup
-    layer_registry = QgsMapLayerRegistry.instance()
+    layer_registry = QgsProject.instance()
     layer_registry.removeAllMapLayers()
 
     impact_function = ImpactFunction()
@@ -213,7 +213,7 @@ def inasafe_multi_exposure_analysis(
     """
     # Clean up layer registry before using
     # In case previous task exited prematurely before cleanup
-    layer_registry = QgsMapLayerRegistry.instance()
+    layer_registry = QgsProject.instance()
     layer_registry.removeAllMapLayers()
 
     multi_exposure_if = MultiExposureImpactFunction()
@@ -324,7 +324,7 @@ def generate_report(
     """
     # Clean up layer registry before using
     # In case previous task exited prematurely before cleanup
-    layer_registry = QgsMapLayerRegistry.instance()
+    layer_registry = QgsProject.instance()
     layer_registry.removeAllMapLayers()
 
     output_metadata = read_iso19115_metadata(impact_layer_uri)
@@ -344,14 +344,14 @@ def generate_report(
         root = QgsProject.instance().layerTreeRoot()
 
         group_analysis = root.insertGroup(0, impact_function.name)
-        group_analysis.setVisible(True)
+        group_analysis.setItemVisibilityChecked(True)
         group_analysis.setCustomProperty(
             MULTI_EXPOSURE_ANALYSIS_FLAG, True)
 
         for layer in impact_function.outputs:
-            QgsMapLayerRegistry.instance().addMapLayer(layer, False)
+            QgsProject.instance().addMapLayer(layer, False)
             layer_node = group_analysis.addLayer(layer)
-            layer_node.setVisible(False)
+            layer_node.setItemVisibilityChecked(False)
 
             # set layer title if any
             try:
@@ -362,7 +362,7 @@ def generate_report(
 
         for analysis in impact_function.impact_functions:
             detailed_group = group_analysis.insertGroup(0, analysis.name)
-            detailed_group.setVisible(True)
+            detailed_group.setItemVisibilityChecked(True)
             add_impact_layers_to_canvas(analysis, group=detailed_group)
     else:
         impact_function = (
@@ -515,7 +515,7 @@ def push_to_geonode(layer_uri):
         'username': REALTIME_GEONODE_USER,
         'password': REALTIME_GEONODE_PASSWORD
     }
-    for key, value in requirements.items():
+    for key, value in list(requirements.items()):
         if not value:
             message = 'Can not upload to geonode because the %s is empty' % key
             LOGGER.warning(message)

src/headless/tasks/inasafe_wrapper.py

@@ -1,6 +1,6 @@
 # coding=utf-8
 """Task for InaSAFE Headless."""
-
+from importlib import reload
 from headless.celery_app import app, start_inasafe
 from headless.tasks import inasafe_analysis
 from headless.utils import get_headless_logger

src/headless/tasks/test/data/input_layers/buildings.xml

@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="UTF-8"?>
+<?xml version="1.0" ?>
 <gmd:MD_Metadata xmlns:gco="http://www.isotc211.org/2005/gco" xmlns:gmd="http://www.isotc211.org/2005/gmd" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.isotc211.org/2005/gmd http://www.isotc211.org/2005/gmd/gmd.xsd">
   <gmd:fileIdentifier>
     <gco:CharacterString>None</gco:CharacterString>
@@ -171,57 +171,48 @@
       </gmd:extent>
       <gmd:supplementalInformation>
         <inasafe>
-          <classification>
-            <gco:CharacterString>generic_structure_classes</gco:CharacterString>
-          </classification>
-          <inasafe_fields>
-            <gco:Dictionary>{&quot;exposure_id_field&quot;: &quot;exposure_id&quot;, &quot;exposure_type_field&quot;: &quot;exposure_type&quot;}</gco:Dictionary>
-          </inasafe_fields>
-          <inasafe_default_values>
-            <gco:Dictionary/>
-          </inasafe_default_values>
+          <layer_purpose>
+            <gco:CharacterString>exposure</gco:CharacterString>
+          </layer_purpose>
+          <layer_mode>
+            <gco:CharacterString>classified</gco:CharacterString>
+          </layer_mode>
+          <layer_geometry>
+            <gco:CharacterString>polygon</gco:CharacterString>
+          </layer_geometry>
+          <keyword_version>
+            <gco:CharacterString>5.0</gco:CharacterString>
+          </keyword_version>
           <scale>
             <gco:CharacterString/>
           </scale>
-          <multipart_polygon>
-            <gco:Boolean/>
-          </multipart_polygon>
           <source>
             <gco:CharacterString>Test layer</gco:CharacterString>
           </source>
-          <value_map>
-            <gco:Dictionary>{&quot;commercial&quot;: [&quot;shop&quot;], &quot;education&quot;: [&quot;school&quot;], &quot;health&quot;: [&quot;hospital&quot;], &quot;government&quot;: [&quot;ministry&quot;]}</gco:Dictionary>
-          </value_map>
-          <layer_geometry>
-            <gco:CharacterString>polygon</gco:CharacterString>
-          </layer_geometry>
+          <inasafe_fields>
+            <gco:Dictionary>{&quot;exposure_id_field&quot;: &quot;exposure_id&quot;, &quot;exposure_type_field&quot;: &quot;exposure_type&quot;}</gco:Dictionary>
+          </inasafe_fields>
+          <inasafe_default_values>
+            <gco:Dictionary/>
+          </inasafe_default_values>
+          <extra_keywords>
+            <gco:Dictionary/>
+          </extra_keywords>
           <exposure>
             <gco:CharacterString>structure</gco:CharacterString>
           </exposure>
-          <report>
-            <gco:CharacterString/>
-          </report>
           <exposure_unit>
             <gco:CharacterString/>
           </exposure_unit>
-          <keyword_version>
-            <gco:CharacterString>4.0</gco:CharacterString>
-          </keyword_version>
-          <datatype>
-            <gco:CharacterString/>
-          </datatype>
-          <allow_resampling>
-            <gco:CharacterString/>
-          </allow_resampling>
-          <layer_purpose>
-            <gco:CharacterString>exposure</gco:CharacterString>
-          </layer_purpose>
-          <resolution>
-            <gco:FloatTuple/>
-          </resolution>
-          <layer_mode>
-            <gco:CharacterString>classified</gco:CharacterString>
-          </layer_mode>
+          <classification>
+            <gco:CharacterString>generic_structure_classes</gco:CharacterString>
+          </classification>
+          <value_map>
+            <gco:Dictionary>{&quot;commercial&quot;: [&quot;shop&quot;], &quot;education&quot;: [&quot;school&quot;], &quot;health&quot;: [&quot;hospital&quot;], &quot;government&quot;: [&quot;ministry&quot;]}</gco:Dictionary>
+          </value_map>
+          <active_band>
+            <gco:Integer/>
+          </active_band>
         </inasafe>
       </gmd:supplementalInformation>
     </gmd:MD_DataIdentification>