From c7ca779338a91d07847a088f4f8558698a0fc481 Mon Sep 17 00:00:00 2001 From: Rizky Maulana Nugraha Date: Tue, 28 Jul 2020 17:40:29 +0700 Subject: [PATCH 1/3] Apply futurize to maintain compatibility --- src/headless/celery_app.py | 2 +- src/headless/settings.py | 1 + src/headless/tasks/inasafe_analysis.py | 4 +- src/headless/tasks/test/helpers.py | 1 + src/headless/tasks/test/test_celery_task.py | 1 + .../tasks/test/test_generate_report.py | 46 ++++++++++--------- src/headless/tasks/test/test_run_analysis.py | 29 ++++++------ 7 files changed, 45 insertions(+), 39 deletions(-) diff --git a/src/headless/celery_app.py b/src/headless/celery_app.py index 844b438..d7ccb6f 100644 --- a/src/headless/celery_app.py +++ b/src/headless/celery_app.py @@ -30,7 +30,7 @@ def load_inasafe_settings(): from safe.definitions import default_settings from safe.utilities.settings import set_setting - for key, value in default_settings.inasafe_default_settings.iteritems(): + for key, value in list(default_settings.inasafe_default_settings.items()): set_setting(key, value) # Override settings from INASAFE_SETTINGS_PATH diff --git a/src/headless/settings.py b/src/headless/settings.py index 080e120..32c67a9 100644 --- a/src/headless/settings.py +++ b/src/headless/settings.py @@ -1,5 +1,6 @@ # coding=utf-8 """InaSAFE Headless settings.""" +from builtins import str import logging from distutils.util import strtobool diff --git a/src/headless/tasks/inasafe_analysis.py b/src/headless/tasks/inasafe_analysis.py index 240b371..4ec67eb 100644 --- a/src/headless/tasks/inasafe_analysis.py +++ b/src/headless/tasks/inasafe_analysis.py @@ -55,7 +55,7 @@ def clean_metadata(metadata): :param metadata: Metadata as dictionary. :type metadata: dict """ - for key, value in metadata.items(): + for key, value in list(metadata.items()): if isinstance(value, dict): clean_metadata(value) if isinstance(value, QUrl): @@ -515,7 +515,7 @@ def push_to_geonode(layer_uri): 'username': REALTIME_GEONODE_USER, 'password': REALTIME_GEONODE_PASSWORD } - for key, value in requirements.items(): + for key, value in list(requirements.items()): if not value: message = 'Can not upload to geonode because the %s is empty' % key LOGGER.warning(message) diff --git a/src/headless/tasks/test/helpers.py b/src/headless/tasks/test/helpers.py index bf08f04..48b7224 100644 --- a/src/headless/tasks/test/helpers.py +++ b/src/headless/tasks/test/helpers.py @@ -1,4 +1,5 @@ # coding=utf-8 +from builtins import range import os from functools import wraps diff --git a/src/headless/tasks/test/test_celery_task.py b/src/headless/tasks/test/test_celery_task.py index 25be26d..baffa33 100644 --- a/src/headless/tasks/test/test_celery_task.py +++ b/src/headless/tasks/test/test_celery_task.py @@ -1,5 +1,6 @@ # coding=utf-8 """Unit test for celery task.""" +from past.builtins import basestring import os import pickle import unittest diff --git a/src/headless/tasks/test/test_generate_report.py b/src/headless/tasks/test/test_generate_report.py index bd96fd7..ecf7ac8 100644 --- a/src/headless/tasks/test/test_generate_report.py +++ b/src/headless/tasks/test/test_generate_report.py @@ -1,4 +1,6 @@ # coding=utf-8 +from __future__ import print_function +from past.builtins import basestring import os import unittest from distutils.util import strtobool @@ -52,7 +54,7 @@ def test_generate_report_qlr(self): self.assertEqual(ANALYSIS_SUCCESS, result['status'], result['message']) self.assertLess(0, len(result['output'])) - for key, layer_uri in result['output'].items(): + for key, layer_uri in list(result['output'].items()): self.assertTrue(os.path.exists(layer_uri)) self.assertTrue(layer_uri.startswith(OUTPUT_DIRECTORY)) @@ -71,8 +73,8 @@ def test_generate_report_qlr(self): result = async_result.get() self.assertEqual( ImpactReport.REPORT_GENERATION_SUCCESS, result['status']) - for key, products in result['output'].items(): - for product_key, product_uri in products.items(): + for key, products in list(result['output'].items()): + for product_key, product_uri in list(products.items()): message = 'Product %s is not found in %s' % ( product_key, product_uri) self.assertTrue(os.path.exists(product_uri), message) @@ -88,7 +90,7 @@ def test_generate_report_with_aggregation(self): result = result_delay.get() self.assertEqual(ANALYSIS_SUCCESS, result['status'], result['message']) self.assertLess(0, len(result['output'])) - for key, layer_uri in result['output'].items(): + for key, layer_uri in list(result['output'].items()): self.assertTrue(os.path.exists(layer_uri)) self.assertTrue(layer_uri.startswith(OUTPUT_DIRECTORY)) @@ -107,8 +109,8 @@ def test_generate_report_with_aggregation(self): result = async_result.get() self.assertEqual( ImpactReport.REPORT_GENERATION_SUCCESS, result['status']) - for key, products in result['output'].items(): - for product_key, product_uri in products.items(): + for key, products in list(result['output'].items()): + for product_key, product_uri in list(products.items()): message = 'Product %s is not found in %s' % ( product_key, product_uri) self.assertTrue(os.path.exists(product_uri), message) @@ -124,7 +126,7 @@ def test_generate_report_without_aggregation(self): result = result_delay.get() self.assertEqual(0, result['status'], result['message']) self.assertLess(0, len(result['output'])) - for key, layer_uri in result['output'].items(): + for key, layer_uri in list(result['output'].items()): self.assertTrue(os.path.exists(layer_uri)) self.assertTrue(layer_uri.startswith(OUTPUT_DIRECTORY)) @@ -143,8 +145,8 @@ def test_generate_report_without_aggregation(self): result = async_result.get() self.assertEqual( ImpactReport.REPORT_GENERATION_SUCCESS, result['status']) - for key, products in result['output'].items(): - for product_key, product_uri in products.items(): + for key, products in list(result['output'].items()): + for product_key, product_uri in list(products.items()): message = 'Product %s is not found in %s' % ( product_key, product_uri) self.assertTrue(os.path.exists(product_uri), message) @@ -164,13 +166,13 @@ def test_generate_multi_exposure_report(self): self.assertEqual(ANALYSIS_SUCCESS, result['status'], result['message']) self.assertLess(0, len(result['output'])) num_exposure_output = 0 - for key, layer_uri in result['output'].items(): + for key, layer_uri in list(result['output'].items()): if isinstance(layer_uri, basestring): self.assertTrue(os.path.exists(layer_uri)) self.assertTrue(layer_uri.startswith(OUTPUT_DIRECTORY)) elif isinstance(layer_uri, dict): num_exposure_output += 1 - for the_key, the_layer_uri in layer_uri.items(): + for the_key, the_layer_uri in list(layer_uri.items()): self.assertTrue(os.path.exists(the_layer_uri)) self.assertTrue(the_layer_uri.startswith(OUTPUT_DIRECTORY)) @@ -189,8 +191,8 @@ def test_generate_multi_exposure_report(self): result = async_result.get() self.assertEqual( ImpactReport.REPORT_GENERATION_SUCCESS, result['status']) - for key, products in result['output'].items(): - for product_key, product_uri in products.items(): + for key, products in list(result['output'].items()): + for product_key, product_uri in list(products.items()): message = 'Product %s is not found in %s' % ( product_key, product_uri) self.assertTrue(os.path.exists(product_uri), message) @@ -207,7 +209,7 @@ def test_generate_custom_report(self): result = result_delay.get() self.assertEqual(ANALYSIS_SUCCESS, result['status'], result['message']) self.assertLess(0, len(result['output'])) - for key, layer_uri in result['output'].items(): + for key, layer_uri in list(result['output'].items()): self.assertTrue(os.path.exists(layer_uri)) self.assertTrue(layer_uri.startswith(OUTPUT_DIRECTORY)) @@ -222,14 +224,14 @@ def test_generate_custom_report(self): self.assertEqual( ImpactReport.REPORT_GENERATION_SUCCESS, result['status']) product_keys = [] - for key, products in result['output'].items(): - for product_key, product_uri in products.items(): + for key, products in list(result['output'].items()): + for product_key, product_uri in list(products.items()): product_keys.append(product_key) message = 'Product %s is not found in %s' % ( product_key, product_uri) self.assertTrue(os.path.exists(product_uri), message) if custom_map_template_basename == product_key: - print product_uri + print(product_uri) # Check if custom map template found. self.assertIn(custom_map_template_basename, product_keys) @@ -250,7 +252,7 @@ def test_generate_report_with_basemap(self): self.assertEqual(ANALYSIS_SUCCESS, result['status'], result['message']) self.assertLess(0, len(result['output'])) - for key, layer_uri in result['output'].items(): + for key, layer_uri in list(result['output'].items()): self.assertTrue(os.path.exists(layer_uri)) self.assertTrue(layer_uri.startswith(OUTPUT_DIRECTORY)) @@ -275,14 +277,14 @@ def test_generate_report_with_basemap(self): self.assertEqual( ImpactReport.REPORT_GENERATION_SUCCESS, result['status']) product_keys = [] - for key, products in result['output'].items(): - for product_key, product_uri in products.items(): + for key, products in list(result['output'].items()): + for product_key, product_uri in list(products.items()): product_keys.append(product_key) message = 'Product %s is not found in %s' % ( product_key, product_uri) self.assertTrue(os.path.exists(product_uri), message) if custom_map_template_basename == product_key: - print product_uri + print(product_uri) @retry_on_worker_lost_error() def test_get_generated_report(self): @@ -293,7 +295,7 @@ def test_get_generated_report(self): result = result_delay.get() self.assertEqual(ANALYSIS_SUCCESS, result['status'], result['message']) self.assertLess(0, len(result['output'])) - for key, layer_uri in result['output'].items(): + for key, layer_uri in list(result['output'].items()): self.assertTrue(os.path.exists(layer_uri)) self.assertTrue(layer_uri.startswith(OUTPUT_DIRECTORY)) diff --git a/src/headless/tasks/test/test_run_analysis.py b/src/headless/tasks/test/test_run_analysis.py index 0a96f86..acb94d9 100644 --- a/src/headless/tasks/test/test_run_analysis.py +++ b/src/headless/tasks/test/test_run_analysis.py @@ -1,4 +1,5 @@ # coding=utf-8 +from past.builtins import basestring import os import unittest from distutils.util import strtobool @@ -39,7 +40,7 @@ def test_run_analysis(self): result = result_delay.get() self.assertEqual(ANALYSIS_SUCCESS, result['status'], result['message']) self.assertLess(0, len(result['output'])) - for key, layer_uri in result['output'].items(): + for key, layer_uri in list(result['output'].items()): self.assertTrue(os.path.exists(layer_uri)) self.assertTrue(layer_uri.startswith(OUTPUT_DIRECTORY)) @@ -49,7 +50,7 @@ def test_run_analysis(self): result = result_delay.get() self.assertEqual(ANALYSIS_SUCCESS, result['status'], result['message']) self.assertLess(0, len(result['output'])) - for key, layer_uri in result['output'].items(): + for key, layer_uri in list(result['output'].items()): self.assertTrue(os.path.exists(layer_uri)) self.assertTrue(layer_uri.startswith(OUTPUT_DIRECTORY)) @@ -68,13 +69,13 @@ def test_run_multi_exposure_analysis(self): self.assertEqual(ANALYSIS_SUCCESS, result['status'], result['message']) self.assertLess(0, len(result['output'])) num_exposure_output = 0 - for key, layer_uri in result['output'].items(): + for key, layer_uri in list(result['output'].items()): if isinstance(layer_uri, basestring): self.assertTrue(os.path.exists(layer_uri)) self.assertTrue(layer_uri.startswith(OUTPUT_DIRECTORY)) elif isinstance(layer_uri, dict): num_exposure_output += 1 - for the_key, the_layer_uri in layer_uri.items(): + for the_key, the_layer_uri in list(layer_uri.items()): self.assertTrue(os.path.exists(the_layer_uri)) self.assertTrue(the_layer_uri.startswith(OUTPUT_DIRECTORY)) # Check the number of per exposure output is the same as the number @@ -88,13 +89,13 @@ def test_run_multi_exposure_analysis(self): self.assertEqual(ANALYSIS_SUCCESS, result['status'], result['message']) self.assertLess(0, len(result['output'])) num_exposure_output = 0 - for key, layer_uri in result['output'].items(): + for key, layer_uri in list(result['output'].items()): if isinstance(layer_uri, basestring): self.assertTrue(os.path.exists(layer_uri)) self.assertTrue(layer_uri.startswith(OUTPUT_DIRECTORY)) elif isinstance(layer_uri, dict): num_exposure_output += 1 - for the_key, the_layer_uri in layer_uri.items(): + for the_key, the_layer_uri in list(layer_uri.items()): self.assertTrue(os.path.exists(the_layer_uri)) self.assertTrue(the_layer_uri.startswith(OUTPUT_DIRECTORY)) # Check the number of per exposure output is the same as the number @@ -115,7 +116,7 @@ def test_run_analysis_qlr(self): self.assertEqual(ANALYSIS_SUCCESS, result['status'], result['message']) self.assertLess(0, len(result['output'])) - for key, layer_uri in result['output'].items(): + for key, layer_uri in list(result['output'].items()): self.assertTrue(os.path.exists(layer_uri)) self.assertTrue(layer_uri.startswith(OUTPUT_DIRECTORY)) @@ -126,7 +127,7 @@ def test_run_analysis_qlr(self): self.assertEqual(ANALYSIS_SUCCESS, result['status'], result['message']) self.assertLess(0, len(result['output'])) - for key, layer_uri in result['output'].items(): + for key, layer_uri in list(result['output'].items()): self.assertTrue(os.path.exists(layer_uri)) self.assertTrue(layer_uri.startswith(OUTPUT_DIRECTORY)) @@ -140,7 +141,7 @@ def test_run_multilingual_analysis(self): result = result_delay.get() self.assertEqual(ANALYSIS_SUCCESS, result['status'], result['message']) self.assertLess(0, len(result['output'])) - for key, layer_uri in result['output'].items(): + for key, layer_uri in list(result['output'].items()): self.assertTrue(os.path.exists(layer_uri)) self.assertTrue(layer_uri.startswith(OUTPUT_DIRECTORY)) @@ -161,8 +162,8 @@ def test_run_multilingual_analysis(self): result = async_result.get() self.assertEqual( ImpactReport.REPORT_GENERATION_SUCCESS, result['status']) - for key, products in result['output'].items(): - for product_key, product_uri in products.items(): + for key, products in list(result['output'].items()): + for product_key, product_uri in list(products.items()): message = 'Product %s is not found in %s' % ( product_key, product_uri) self.assertTrue(os.path.exists(product_uri), message) @@ -173,7 +174,7 @@ def test_run_multilingual_analysis(self): result = result_delay.get() self.assertEqual(ANALYSIS_SUCCESS, result['status'], result['message']) self.assertLess(0, len(result['output'])) - for key, layer_uri in result['output'].items(): + for key, layer_uri in list(result['output'].items()): self.assertTrue(os.path.exists(layer_uri)) self.assertTrue(layer_uri.startswith(OUTPUT_DIRECTORY)) @@ -196,8 +197,8 @@ def test_run_multilingual_analysis(self): result = async_result.get() self.assertEqual( ImpactReport.REPORT_GENERATION_SUCCESS, result['status']) - for key, products in result['output'].items(): - for product_key, product_uri in products.items(): + for key, products in list(result['output'].items()): + for product_key, product_uri in list(products.items()): message = 'Product %s is not found in %s' % ( product_key, product_uri) self.assertTrue(os.path.exists(product_uri), message) From 1708e711b9fc6eeee484fad28388cea2e6c91eae Mon Sep 17 00:00:00 2001 From: Rizky Maulana Nugraha Date: Tue, 28 Jul 2020 17:42:20 +0700 Subject: [PATCH 2/3] Address Github alert --- REQUIREMENTS-TRAVIS.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/REQUIREMENTS-TRAVIS.txt b/REQUIREMENTS-TRAVIS.txt index b806965..26ee78e 100644 --- a/REQUIREMENTS-TRAVIS.txt +++ b/REQUIREMENTS-TRAVIS.txt @@ -1,3 +1,3 @@ -ansible==2.3.1.0 +ansible>=2.6.18 codecov flake8==3.6.0 From 0dbf432743a18e7c151b0d85519c5ad2cc419b5d Mon Sep 17 00:00:00 2001 From: Rizky Maulana Nugraha Date: Wed, 29 Jul 2020 00:35:08 +0700 Subject: [PATCH 3/3] Migrate to QGIS 3 - Migrate QLR handling from QgsMapLayer.fromLayerDefinition to QgsLayerDefinition.loadLayerDefinitionLayers - Upgrade docker image to use qgis/qgis:release-3_14 --- .travis.yml | 5 +- .../development/group_vars/all.travis.yml | 2 +- deployment/docker-headless/Dockerfile | 13 +- .../docker-headless/docker-entrypoint.sh | 2 +- .../production/docker/headless/Dockerfile | 8 +- .../docker/headless/docker-entrypoint.sh | 3 +- src/headless/REQUIREMENTS.txt | 7 +- src/headless/celery_app.py | 1 + src/headless/tasks/inasafe_analysis.py | 18 +- src/headless/tasks/inasafe_wrapper.py | 2 +- .../test/data/input_layers/buildings.xml | 71 +++--- .../data/input_layers/buildings_geojson.qlr | 216 ++++++++++++++++++ .../data/input_layers/buildings_geojson.xml | 1 + .../input_layers/population_multi_fields.qml | 2 +- .../test/data/input_layers/small_grid.qml | 2 +- src/headless/tasks/test/helpers.py | 2 +- .../tasks/test/test_generate_report.py | 4 - src/headless/tasks/test/test_run_analysis.py | 4 - .../tasks/test/test_subsequent_run.py | 4 +- src/headless/utils.py | 9 +- 20 files changed, 291 insertions(+), 85 deletions(-) create mode 100644 src/headless/tasks/test/data/input_layers/buildings_geojson.qlr create mode 120000 src/headless/tasks/test/data/input_layers/buildings_geojson.xml diff --git a/.travis.yml b/.travis.yml index bafd753..1744ac4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,7 @@ sudo: required language: python python: - - "2.7" + - "3.6" env: global: @@ -23,9 +23,6 @@ env: HEADLESS_COMMAND=dev SUBSEQUENT_RUN_TESTING=True -virtualenv: - system_site_packages: true - services: - docker diff --git a/deployment/ansible/development/group_vars/all.travis.yml b/deployment/ansible/development/group_vars/all.travis.yml index 01051dc..f1addd6 100644 --- a/deployment/ansible/development/group_vars/all.travis.yml +++ b/deployment/ansible/development/group_vars/all.travis.yml @@ -16,7 +16,7 @@ interpreters: inasafe: repo: https://github.com/inasafe/inasafe.git remote: upstream - version: inasafe_4 + version: maintenance-fix depth: 1 inasafe_headless_worker: diff --git a/deployment/docker-headless/Dockerfile b/deployment/docker-headless/Dockerfile index 79faad2..030e5d2 100644 --- a/deployment/docker-headless/Dockerfile +++ b/deployment/docker-headless/Dockerfile @@ -1,5 +1,5 @@ #--------- Generic stuff all our Dockerfiles should start with so we get caching ------------ -FROM kartoza/qgis-desktop:2.18 +FROM qgis/qgis:release-3_14 RUN apt-get -y update; apt-get -y --force-yes install pwgen git inotify-tools @@ -9,7 +9,10 @@ RUN apt-get -y update; apt-get -y --force-yes install pwgen git inotify-tools RUN apt-get update -y; apt-get install -y --force-yes openssh-server sudo RUN mkdir /var/run/sshd RUN echo 'root:docker' | chpasswd -RUN sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config +# Comment out PermitRootLogin setting, whatever it is +RUN sed -i 's/^PermitRootLogin */#PermitRootLogin /' /etc/ssh/sshd_config +# Write out PermitRootLogin setting at the end +RUN echo "PermitRootLogin yes" >> /etc/ssh/sshd_config # SSH login fix. Otherwise user is kicked off after login RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd @@ -21,8 +24,7 @@ RUN echo "export VISIBLE=now" >> /etc/profile #-------------Application Specific Stuff ---------------------------------------------------- # Install git, xvfb -RUN apt-get -y update; apt-get -y --force-yes install git xvfb python-setuptools python-dev libssl-dev libffi-dev python-scipy -RUN easy_install pip==9.0.1 +RUN apt-get -y update; apt-get -y --force-yes install git xvfb python3-setuptools python3-dev libssl-dev libffi-dev python3-scipy # Copy ubuntu fonts RUN apt-get -y update; apt-get -y --force-yes install wget unzip ADD ubuntu-font-family-0.83.zip /ubuntu-font-family-0.83.zip @@ -31,8 +33,9 @@ RUN mv ubuntu-font-family-0.83 /usr/share/fonts/truetype/ubuntu-font-family RUN fc-cache -f -v # This image instance uses dist-packages directory -ADD sitecustomize.py /usr/local/lib/python2.7/dist-packages/sitecustomize.py +ADD sitecustomize.py /usr/lib/python3/dist-packages/sitecustomize.py ADD REQUIREMENTS.txt /REQUIREMENTS.txt +RUN ln -s /usr/bin/pip3 /usr/bin/pip RUN pip install -r REQUIREMENTS.txt ADD docker-entrypoint.sh /docker-entrypoint.sh diff --git a/deployment/docker-headless/docker-entrypoint.sh b/deployment/docker-headless/docker-entrypoint.sh index 8c461b0..46887da 100644 --- a/deployment/docker-headless/docker-entrypoint.sh +++ b/deployment/docker-headless/docker-entrypoint.sh @@ -2,7 +2,7 @@ # Wait run xvfb while [ -z "$(pidof /usr/bin/Xvfb)" ]; do - start-stop-daemon --start -b -x /usr/bin/Xvfb ${DISPLAY} + start-stop-daemon --start -b -x /usr/bin/Xvfb ${DISPLAY} -- -screen 0 1024x768x24 -ac +extension GLX +render -noreset -nolisten tcp sleep 5 done diff --git a/deployment/production/docker/headless/Dockerfile b/deployment/production/docker/headless/Dockerfile index 53abbf5..bab377c 100644 --- a/deployment/production/docker/headless/Dockerfile +++ b/deployment/production/docker/headless/Dockerfile @@ -1,12 +1,11 @@ #--------- Generic stuff all our Dockerfiles should start with so we get caching ------------ -FROM kartoza/qgis-desktop:2.18 +FROM qgis/qgis:release-3_14 RUN apt-get -y update; apt-get -y --force-yes install pwgen git inotify-tools #-------------Application Specific Stuff ---------------------------------------------------- # Install git, xvfb -RUN apt-get -y update; apt-get -y --force-yes install git xvfb python-setuptools python-dev libssl-dev libffi-dev python-scipy -RUN easy_install pip==9.0.1 +RUN apt-get -y update; apt-get -y --force-yes install git xvfb python3-setuptools python3-dev libssl-dev libffi-dev python3-scipy # Copy ubuntu fonts RUN apt-get -y update; apt-get -y --force-yes install wget unzip ADD ubuntu-font-family-0.83.zip /ubuntu-font-family-0.83.zip @@ -15,6 +14,7 @@ RUN mv ubuntu-font-family-0.83 /usr/share/fonts/truetype/ubuntu-font-family RUN fc-cache -f -v ADD REQUIREMENTS.txt /REQUIREMENTS.txt +RUN ln -s /usr/bin/pip3 /usr/bin/pip RUN pip install -r REQUIREMENTS.txt ADD docker-entrypoint.sh /docker-entrypoint.sh @@ -23,7 +23,7 @@ RUN chmod +x /docker-entrypoint.sh # Install InaSAFE Core RUN mkdir -p /usr/share/qgis/python/plugins WORKDIR /usr/share/qgis/python/plugins -ARG INASAFE_CORE_TAG=version-4_4_0 +ARG INASAFE_CORE_TAG=develop RUN git clone --branch ${INASAFE_CORE_TAG} --depth 1 --recursive https://github.com/inasafe/inasafe.git inasafe # Install InaSAFE Headless diff --git a/deployment/production/docker/headless/docker-entrypoint.sh b/deployment/production/docker/headless/docker-entrypoint.sh index c0fafb3..0ca7e37 100644 --- a/deployment/production/docker/headless/docker-entrypoint.sh +++ b/deployment/production/docker/headless/docker-entrypoint.sh @@ -2,11 +2,12 @@ # Wait run xvfb while [ -z "$(pidof /usr/bin/Xvfb)" ]; do - start-stop-daemon --start -b -x /usr/bin/Xvfb ${DISPLAY} + start-stop-daemon --start -b -x /usr/bin/Xvfb ${DISPLAY} -- -screen 0 1024x768x24 -ac +extension GLX +render -noreset -nolisten tcp sleep 5 done cp -n /home/app/headless/celeryconfig_sample.py /home/app/headless/celeryconfig.py +echo "Config file copied" if [ $# -eq 2 ] && [ $1 = "prod" ] && [ $2 = "inasafe-headless-worker" ]; then /usr/local/bin/celery -A headless.celery_app worker -l info -Q inasafe-headless -n inasafe-headless.%h diff --git a/src/headless/REQUIREMENTS.txt b/src/headless/REQUIREMENTS.txt index 76120b4..f0b5969 100644 --- a/src/headless/REQUIREMENTS.txt +++ b/src/headless/REQUIREMENTS.txt @@ -1,8 +1,11 @@ requests>=2.6.2 tzlocal==1.2 -celery==4.1.0 -nosexcover +hammock==0.2.4 nose2==0.6.5 coverage==4.4.2 +threadpool==1.3.2 +pyinotify==0.9.6 +celery==4.1.1 raven==5.29.0 mock==2.0.0 +nose==1.3.7 diff --git a/src/headless/celery_app.py b/src/headless/celery_app.py index d7ccb6f..1f5ce50 100644 --- a/src/headless/celery_app.py +++ b/src/headless/celery_app.py @@ -1,5 +1,6 @@ # coding=utf-8 import importlib +from importlib import reload import json import os diff --git a/src/headless/tasks/inasafe_analysis.py b/src/headless/tasks/inasafe_analysis.py index 4ec67eb..0f1439f 100644 --- a/src/headless/tasks/inasafe_analysis.py +++ b/src/headless/tasks/inasafe_analysis.py @@ -5,10 +5,10 @@ from copy import deepcopy from datetime import datetime -from PyQt4.QtCore import QUrl +from qgis.PyQt.QtCore import QUrl from qgis.core import ( - QgsCoordinateReferenceSystem, QgsMapLayerRegistry, QgsProject) + QgsCoordinateReferenceSystem, QgsProject) from safe.definitions.constants import ( PREPARE_SUCCESS, ANALYSIS_SUCCESS, MULTI_EXPOSURE_ANALYSIS_FLAG) @@ -118,7 +118,7 @@ def inasafe_analysis( """ # Clean up layer registry before using # In case previous task exited prematurely before cleanup - layer_registry = QgsMapLayerRegistry.instance() + layer_registry = QgsProject.instance() layer_registry.removeAllMapLayers() impact_function = ImpactFunction() @@ -213,7 +213,7 @@ def inasafe_multi_exposure_analysis( """ # Clean up layer registry before using # In case previous task exited prematurely before cleanup - layer_registry = QgsMapLayerRegistry.instance() + layer_registry = QgsProject.instance() layer_registry.removeAllMapLayers() multi_exposure_if = MultiExposureImpactFunction() @@ -324,7 +324,7 @@ def generate_report( """ # Clean up layer registry before using # In case previous task exited prematurely before cleanup - layer_registry = QgsMapLayerRegistry.instance() + layer_registry = QgsProject.instance() layer_registry.removeAllMapLayers() output_metadata = read_iso19115_metadata(impact_layer_uri) @@ -344,14 +344,14 @@ def generate_report( root = QgsProject.instance().layerTreeRoot() group_analysis = root.insertGroup(0, impact_function.name) - group_analysis.setVisible(True) + group_analysis.setItemVisibilityChecked(True) group_analysis.setCustomProperty( MULTI_EXPOSURE_ANALYSIS_FLAG, True) for layer in impact_function.outputs: - QgsMapLayerRegistry.instance().addMapLayer(layer, False) + QgsProject.instance().addMapLayer(layer, False) layer_node = group_analysis.addLayer(layer) - layer_node.setVisible(False) + layer_node.setItemVisibilityChecked(False) # set layer title if any try: @@ -362,7 +362,7 @@ def generate_report( for analysis in impact_function.impact_functions: detailed_group = group_analysis.insertGroup(0, analysis.name) - detailed_group.setVisible(True) + detailed_group.setItemVisibilityChecked(True) add_impact_layers_to_canvas(analysis, group=detailed_group) else: impact_function = ( diff --git a/src/headless/tasks/inasafe_wrapper.py b/src/headless/tasks/inasafe_wrapper.py index 07ca7b3..b2c7176 100644 --- a/src/headless/tasks/inasafe_wrapper.py +++ b/src/headless/tasks/inasafe_wrapper.py @@ -1,6 +1,6 @@ # coding=utf-8 """Task for InaSAFE Headless.""" - +from importlib import reload from headless.celery_app import app, start_inasafe from headless.tasks import inasafe_analysis from headless.utils import get_headless_logger diff --git a/src/headless/tasks/test/data/input_layers/buildings.xml b/src/headless/tasks/test/data/input_layers/buildings.xml index fd8ce41..ceaa723 100644 --- a/src/headless/tasks/test/data/input_layers/buildings.xml +++ b/src/headless/tasks/test/data/input_layers/buildings.xml @@ -1,4 +1,4 @@ - + None @@ -171,57 +171,48 @@ - - generic_structure_classes - - - {"exposure_id_field": "exposure_id", "exposure_type_field": "exposure_type"} - - - - + + exposure + + + classified + + + polygon + + + 5.0 + - - - Test layer - - {"commercial": ["shop"], "education": ["school"], "health": ["hospital"], "government": ["ministry"]} - - - polygon - + + {"exposure_id_field": "exposure_id", "exposure_type_field": "exposure_type"} + + + + + + + structure - - - - - 4.0 - - - - - - - - - exposure - - - - - - classified - + + generic_structure_classes + + + {"commercial": ["shop"], "education": ["school"], "health": ["hospital"], "government": ["ministry"]} + + + + diff --git a/src/headless/tasks/test/data/input_layers/buildings_geojson.qlr b/src/headless/tasks/test/data/input_layers/buildings_geojson.qlr new file mode 100644 index 0000000..8b0eb2b --- /dev/null +++ b/src/headless/tasks/test/data/input_layers/buildings_geojson.qlr @@ -0,0 +1,216 @@ + + + + + + + + + + + + 106.6388189694232409 + -6.29597524568388156 + 106.90657529121263281 + -6.11387787165906804 + + buildings_25dafc4f_de52_4c06_94ab_14cd6a301be1 + ./buildings.geojson + + + + buildings + + + GEOGCRS["WGS 84",DATUM["World Geodetic System 1984",ELLIPSOID["WGS 84",6378137,298.257223563,LENGTHUNIT["metre",1]]],PRIMEM["Greenwich",0,ANGLEUNIT["degree",0.0174532925199433]],CS[ellipsoidal,2],AXIS["geodetic latitude (Lat)",north,ORDER[1],ANGLEUNIT["degree",0.0174532925199433]],AXIS["geodetic longitude (Lon)",east,ORDER[2],ANGLEUNIT["degree",0.0174532925199433]],USAGE[SCOPE["unknown"],AREA["World"],BBOX[-90,-180,90,180]],ID["EPSG",4326]] + +proj=longlat +datum=WGS84 +no_defs + 3452 + 4326 + EPSG:4326 + WGS 84 + longlat + EPSG:7030 + true + + + + + + + dataset + + + + + + + + + + 0 + 0 + + + + + false + + + + + ogr + + + + + + + + + + + 1 + 1 + 1 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 0 + 0 + 1 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 0 + + + 0 + generatedlayout + + + + + + + + + diff --git a/src/headless/tasks/test/data/input_layers/buildings_geojson.xml b/src/headless/tasks/test/data/input_layers/buildings_geojson.xml new file mode 120000 index 0000000..933f876 --- /dev/null +++ b/src/headless/tasks/test/data/input_layers/buildings_geojson.xml @@ -0,0 +1 @@ +buildings.xml \ No newline at end of file diff --git a/src/headless/tasks/test/data/input_layers/population_multi_fields.qml b/src/headless/tasks/test/data/input_layers/population_multi_fields.qml index e34221b..f5a2eb3 100644 --- a/src/headless/tasks/test/data/input_layers/population_multi_fields.qml +++ b/src/headless/tasks/test/data/input_layers/population_multi_fields.qml @@ -420,7 +420,7 @@ Enter the name of the function in the "Python Init function" field. An example follows: """ -from PyQt4.QtGui import QWidget +from qgis.PyQt.QtGui import QWidget def my_form_open(dialog, layer, feature): geom = feature.geometry() diff --git a/src/headless/tasks/test/data/input_layers/small_grid.qml b/src/headless/tasks/test/data/input_layers/small_grid.qml index 23b3608..ee5b841 100644 --- a/src/headless/tasks/test/data/input_layers/small_grid.qml +++ b/src/headless/tasks/test/data/input_layers/small_grid.qml @@ -220,7 +220,7 @@ Enter the name of the function in the "Python Init function" field. An example follows: """ -from PyQt4.QtGui import QWidget +from qgis.PyQt.QtGui import QWidget def my_form_open(dialog, layer, feature): geom = feature.geometry() diff --git a/src/headless/tasks/test/helpers.py b/src/headless/tasks/test/helpers.py index 48b7224..e9f9b2d 100644 --- a/src/headless/tasks/test/helpers.py +++ b/src/headless/tasks/test/helpers.py @@ -24,7 +24,7 @@ buildings_layer_uri = os.path.join( dir_path, 'data', 'input_layers', 'buildings.geojson') buildings_layer_qlr_uri = os.path.join( - dir_path, 'data', 'input_layers', 'buildings.qlr') + dir_path, 'data', 'input_layers', 'buildings_geojson.qlr') shapefile_layer_uri = standard_data_path('exposure', 'airports.shp') ascii_layer_uri = standard_data_path('gisv4', 'hazard', 'earthquake.asc') diff --git a/src/headless/tasks/test/test_generate_report.py b/src/headless/tasks/test/test_generate_report.py index ecf7ac8..366985e 100644 --- a/src/headless/tasks/test/test_generate_report.py +++ b/src/headless/tasks/test/test_generate_report.py @@ -3,7 +3,6 @@ from past.builtins import basestring import os import unittest -from distutils.util import strtobool from headless.settings import OUTPUT_DIRECTORY from headless.tasks.inasafe_analysis import ( @@ -38,9 +37,6 @@ class TestGenerateReport(unittest.TestCase): - @unittest.skipIf( - strtobool(os.environ.get('ON_TRAVIS', 'False')), - """Skipped because we don't have remote service QLR anymore.""") @retry_on_worker_lost_error() def test_generate_report_qlr(self): """Test generating report with QLR files.""" diff --git a/src/headless/tasks/test/test_run_analysis.py b/src/headless/tasks/test/test_run_analysis.py index acb94d9..1be76e4 100644 --- a/src/headless/tasks/test/test_run_analysis.py +++ b/src/headless/tasks/test/test_run_analysis.py @@ -2,7 +2,6 @@ from past.builtins import basestring import os import unittest -from distutils.util import strtobool from headless.settings import OUTPUT_DIRECTORY from headless.tasks.inasafe_wrapper import ( @@ -102,9 +101,6 @@ def test_run_multi_exposure_analysis(self): # of exposures self.assertEqual(num_exposure_output, len(exposure_layer_uris)) - @unittest.skipIf( - strtobool(os.environ.get('ON_TRAVIS', 'False')), - """Skipped because we don't have remote service QLR anymore.""") @retry_on_worker_lost_error() def test_run_analysis_qlr(self): """Test running analysis with QLR files.""" diff --git a/src/headless/tasks/test/test_subsequent_run.py b/src/headless/tasks/test/test_subsequent_run.py index 138f87b..33a2fa2 100644 --- a/src/headless/tasks/test/test_subsequent_run.py +++ b/src/headless/tasks/test/test_subsequent_run.py @@ -2,7 +2,7 @@ import os import unittest -from qgis.core import QgsMapLayerRegistry +from qgis.core import QgsProject from headless.celeryconfig import task_always_eager from headless.tasks.inasafe_wrapper import ( @@ -32,7 +32,7 @@ class TestSubsequentRun(unittest.TestCase): def check_layer_registry_empty(self): # Layer registry should be empty between run - layer_registry = QgsMapLayerRegistry.instance() + layer_registry = QgsProject.instance() self.assertDictEqual(layer_registry.mapLayers(), {}) @unittest.skipUnless( diff --git a/src/headless/utils.py b/src/headless/utils.py index 8ff94b7..8bd4153 100644 --- a/src/headless/utils.py +++ b/src/headless/utils.py @@ -2,7 +2,7 @@ import logging import os -from qgis.core import QgsMapLayer +from qgis.core import QgsLayerDefinition from headless import settings as headless_settings from safe.common.exceptions import NoKeywordsFoundError @@ -50,11 +50,12 @@ def load_layer(full_layer_uri_string, name=None, provider=None): base, ext = os.path.splitext(full_layer_uri_string) if ext.lower() == '.qlr': - layer = QgsMapLayer.fromLayerDefinitionFile(full_layer_uri_string) - if not layer: + layers = QgsLayerDefinition.loadLayerDefinitionLayers( + full_layer_uri_string) + if not layers: return None, None - layer = layer[0] + layer = layers[0] if layer.isValid(): keyword_io = KeywordIO()