Upgrade to Ubuntu 22.04, Python 3.10

Vagrantfile

@@ -54,7 +54,7 @@ MMW_EXTRA_VARS = {
 }
 
 Vagrant.configure("2") do |config|
-  config.vm.box = "bento/ubuntu-20.04"
+  config.vm.box = "bento/ubuntu-22.04"
 
   config.vm.define "services" do |services|
     services.vm.hostname = "services"

deployment/ansible/group_vars/all

@@ -17,16 +17,16 @@ postgresql_password: mmw
 postgresql_database: mmw
 
 postgresql_version: "13"
-postgresql_package_version: "13.*.pgdg20.04+1"
+postgresql_package_version: "13.*.pgdg22.04+1"
 postgresql_support_repository_channel: "main"
-postgresql_support_libpq_version: "13.*.pgdg20.04+1"
+postgresql_support_libpq_version: "13.*.pgdg22.04+1"
 postgresql_support_psycopg2_version: "2.8.*"
 postgis_version: "3"
-postgis_package_version: "3.4*pgdg20.04+1"
+postgis_package_version: "3.4*pgdg22.04+1"
 
 daemontools_version: "1:0.76-7"
 
-python_version: "3.8.*"
+python_version: "3.10.*"
 ansible_python_interpreter: "/usr/bin/python3"
 
 yarn_version: "1.19.*"
@@ -39,9 +39,10 @@ java_version: "8u*"
 java_major_version: "8"
 java_flavor: "openjdk"
 
-docker_version: "5:19.*"
+docker_version: "5:26.*"
+docker_containerd_version: "1.6.*"
 docker_python_version: "4.4.*"
-docker_compose_version: "1.26.*"
+docker_compose_version: "1.29.*"
 
 geop_host: "localhost"
 geop_port: 8090
@@ -55,8 +56,7 @@ nginx_cache_dir: "/var/cache/nginx"
 
 enabled_features: ''
 
-llvmlite_version: "0.37.0"
-numba_version: "0.54.0"
-phantomjs_version: "2.1.*"
+llvmlite_version: "0.38.*"
+numba_version: "0.55.*"
 
-redis_version: "5:5.0.*"
+redis_version: "5:6.0.*"

deployment/ansible/roles.yml

@@ -16,8 +16,6 @@
   version: 0.3.0
 - src: azavea.redis
   version: 0.1.0
-- src: azavea.phantomjs
-  version: 0.1.0
 - src: azavea.build-essential
   version: 0.1.0
 - src: azavea.java

deployment/ansible/roles/model-my-watershed.app/meta/main.yml

@@ -3,6 +3,5 @@ dependencies:
   - { role: "model-my-watershed.base" }
   - { role: "azavea.yarn" }
   - { role: "azavea.nodejs", nodejs_version: "{{ app_nodejs_version }}", nodejs_npm_version: "{{ app_nodejs_npm_version }}" }
-  - { role: "azavea.phantomjs" }
   - { role: "model-my-watershed.celery" }
-  - { role: "azavea.nginx", nginx_delete_default_site: True }
+  - { role: "model-my-watershed.nginx", nginx_delete_default_site: True }

deployment/ansible/roles/model-my-watershed.app/tasks/dev-and-test-dependencies.yml

@@ -1,6 +1,6 @@
 ---
-- name: Install Firefox for UI tests
-  apt: pkg="firefox" state=present
+- name: Install Chromium for UI tests
+  apt: pkg="chromium-browser" state=present
 
 - name: Install Xvfb for JavaScript tests
-  apt: pkg="xvfb=2:1.20.*" state=present
+  apt: pkg="xvfb" state=present

deployment/ansible/roles/model-my-watershed.base/tasks/dependencies.yml

@@ -1,10 +1,10 @@
 ---
 - name: Install Geospatial libraries
   apt:
-    pkg: ["binutils=2.34*",
-          "libproj-dev=6.3.*",
-          "gdal-bin=3.0.*",
-          "libgdal-dev=3.0.*"]
+    pkg: ["binutils=2.38*",
+          "libproj-dev=8.2.*",
+          "gdal-bin=3.4.*",
+          "libgdal-dev=3.4.*"]
     state: present
   when: "['tile-servers'] | is_not_in(group_names)"
 

deployment/ansible/roles/model-my-watershed.celery/defaults/main.yml

@@ -1,2 +1,2 @@
 ---
-celery_version: 5.2.7
+celery_version: 5.4.0

deployment/ansible/roles/model-my-watershed.docker/tasks/main.yml

@@ -1,5 +1,6 @@
 ---
-- name: Install Docker Compose
-  pip:
-    name: docker-compose
-    version: "{{ docker_compose_version }}"
+- name: Add mmw user to docker group
+  user: name=mmw
+        append=yes
+        groups=docker
+        state=present

deployment/ansible/roles/model-my-watershed.geoprocessing/meta/main.yml

@@ -2,4 +2,4 @@
 dependencies:
   - { role: "azavea.java" }
   - { role: "model-my-watershed.base" }
-  - { role: "azavea.nginx", nginx_delete_default_site: True }
+  - { role: "model-my-watershed.nginx", nginx_delete_default_site: True }

deployment/ansible/roles/model-my-watershed.geoprocessing/templates/systemd-geoprocessing.service.j2

@@ -10,9 +10,6 @@ Environment="JAVA_WITH_ENV=/usr/bin/envdir /etc/mmw.d/env /usr/bin/java"
 User=mmw
 WorkingDirectory={{ geop_home }}
 ExecStart=/bin/sh -c '${JAVA_WITH_ENV} -jar mmw-geoprocessing-{{ geop_version }}.jar'
-StandardOutput=syslog
-StandardError=syslog
-SyslogIdentifier=geoprocessing
 
 [Install]
 {% if ['development', 'test'] | some_are_in(group_names) -%}

deployment/ansible/roles/model-my-watershed.nginx/defaults/main.yml

@@ -0,0 +1,3 @@
+---
+nginx_version: "stable"
+nginx_delete_default_site: False

deployment/ansible/roles/model-my-watershed.nginx/handlers/main.yml

@@ -0,0 +1,3 @@
+---
+- name: Restart Nginx
+  service: name=nginx state=restarted

deployment/ansible/roles/model-my-watershed.nginx/tasks/main.yml

@@ -0,0 +1,27 @@
+---
+- name: Install Nginx
+  apt: pkg=nginx-full state=present
+
+- name: Delete default site
+  file: path=/etc/nginx/sites-enabled/default state=absent
+  register: delete_default_site
+  when: nginx_delete_default_site | bool
+  notify:
+    - Restart Nginx
+
+- name: Delete default web root
+  file: path=/var/www/html state=absent
+  when: nginx_delete_default_site | bool and delete_default_site is changed
+
+- name: Check Nginx Upstart service definition exists
+  stat: path=/etc/init/nginx.conf
+  register: nginx_upstart
+
+- name: Configure Nginx log rotation
+  template: src=logrotate_nginx.j2 dest=/etc/logrotate.d/nginx
+  when: nginx_upstart.stat.exists
+
+- name: Configure Nginx
+  template: src=nginx.conf.j2 dest=/etc/nginx/nginx.conf
+  notify:
+    - Restart Nginx

deployment/ansible/roles/model-my-watershed.nginx/templates/logrotate_nginx.j2

@@ -0,0 +1,18 @@
+/var/log/nginx/*.log {
+	weekly
+	missingok
+	rotate 52
+	compress
+	delaycompress
+	notifempty
+	create 0640 www-data adm
+	sharedscripts
+	prerotate
+		if [ -d /etc/logrotate.d/httpd-prerotate ]; then \
+			run-parts /etc/logrotate.d/httpd-prerotate; \
+		fi \
+	endscript
+	postrotate
+		service nginx rotate >/dev/null 2>&1
+	endscript
+}

deployment/ansible/roles/model-my-watershed.nginx/templates/nginx.conf.j2

@@ -0,0 +1,34 @@
+user www-data;
+worker_processes {{ ansible_processor_count }};
+pid /run/nginx.pid;
+
+events {
+	worker_connections 768;
+}
+
+http {
+	sendfile on;
+	tcp_nopush on;
+	tcp_nodelay on;
+	keepalive_timeout 65;
+	types_hash_max_size 2048;
+
+	server_tokens off;
+
+	include /etc/nginx/mime.types;
+	default_type application/octet-stream;
+
+	access_log /var/log/nginx/access.log;
+	error_log /var/log/nginx/error.log;
+
+	gzip on;
+	gzip_vary on;
+	gzip_proxied any;
+	gzip_comp_level 6;
+	gzip_buffers 16 8k;
+	gzip_http_version 1.1;
+	gzip_types text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript application/javascript;
+
+	include /etc/nginx/conf.d/*.conf;
+	include /etc/nginx/sites-enabled/*;
+}

deployment/ansible/roles/model-my-watershed.rwd/handlers/main.yml

@@ -0,0 +1,3 @@
+---
+- name: Restart RWD service
+  service: name=mmw-rwd state=restarted

deployment/ansible/roles/model-my-watershed.rwd/tasks/app.yml

@@ -4,21 +4,10 @@
         state=directory
 
 - name: Configure RWD service definition
-  docker_compose:
-    project_name: mmw
-    state: present
-    definition:
-      version: "3"
-      services:
-        mmw-rwd:
-          image: "{{ rwd_docker_image }}"
-          ports:
-            - "{{ rwd_port }}:{{ rwd_port }}"
-          volumes:
-            - "{{ rwd_data_path }}:{{ rwd_data_path }}:ro"
-          restart: always
-          userns_mode: "host"
-          logging:
-            driver: syslog
-            options:
-              tag: mmw-rwd
+  template: src=systemd-rwd.service.j2
+            dest=/etc/systemd/system/mmw-rwd.service
+  notify:
+    - Restart RWD service
+
+- name: Enable RWD service
+  systemd: name=mmw-rwd.service enabled=yes state=started daemon_reload=yes

deployment/ansible/roles/model-my-watershed.rwd/templates/systemd-rwd.service.j2

@@ -0,0 +1,15 @@
+[Unit]
+Description=mmw-rwd
+After=network.target
+
+[Service]
+User=mmw
+WorkingDirectory={{ rwd_data_path }}
+ExecStart=docker run --rm --volume "{{ rwd_data_path }}:{{ rwd_data_path }}:ro" --publish "{{ rwd_port }}:{{ rwd_port }}" "{{ rwd_docker_image }}"
+
+[Install]
+{% if ['development', 'test'] | some_are_in(group_names) -%}
+WantedBy=opt-app.mount
+{% else %}
+WantedBy=multi-user.target
+{% endif %}

deployment/ansible/roles/model-my-watershed.tiler/meta/main.yml

@@ -2,4 +2,4 @@
 dependencies:
   - { role: "model-my-watershed.base" }
   - { role: "azavea.nodejs", nodejs_version: "{{ tiler_nodejs_version }}", nodejs_npm_version: "{{ tiler_nodejs_npm_version }}" }
-  - { role: "azavea.nginx", nginx_delete_default_site: True }
+  - { role: "model-my-watershed.nginx", nginx_delete_default_site: True }

deployment/ansible/roles/model-my-watershed.tiler/tasks/dependencies.yml

@@ -2,20 +2,20 @@
 - name: Install canvas rendering dependencies
   apt:
     pkg: ["libcairo2-dev=1.16.*",
-          "libpango1.0-dev=1.44.*",
+          "libpango1.0-dev=1.50.*",
           "libjpeg8-dev=8c-*",
           "libgif-dev=5.1.*"]
     state: present
 
 - name: Install Mapnik dependencies
   apt:
-    pkg: ["libmapnik3.0=3.0.*",
-          "libmapnik-dev=3.0.*",
-          "mapnik-utils=3.0.*"]
+    pkg: ["libmapnik3.1=3.1.*",
+          "libmapnik-dev=3.1.*",
+          "mapnik-utils=3.1.*"]
     state: present
 
 - name: Install Windshaft JavaScript dependencies
-  command: sudo npm install --unsafe-perm
+  command: npm install --unsafe-perm
   args:
     chdir: "{{ tiler_home }}"
-  become: False
+  become: True

deployment/cfn/data_plane.py

@@ -195,7 +195,7 @@ def get_recent_bastion_ami(self):
             bastion_ami_id = self.get_input('BastionHostAMI')
         except MKUnresolvableInputError:
             filters = {'name':
-                       'ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-*',
+                       'ubuntu/images/hvm-ssd/ubuntu-jammy-22.04-amd64-server-*',
                        'architecture': 'x86_64',
                        'block-device-mapping.volume-type': 'gp2',
                        'root-device-type': 'ebs',

deployment/packer/driver.py

@@ -18,7 +18,7 @@
 def get_recent_ubuntu_ami(region, aws_profile):
     """Gets AMI ID for current release in region"""
     filters = {
-        'name': 'ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-*',
+        'name': 'ubuntu/images/hvm-ssd/ubuntu-jammy-22.04-amd64-server-*',
         'architecture': 'x86_64',
         'root-device-type': 'ebs',
         'virtualization-type': 'hvm',
@@ -59,7 +59,9 @@ def get_git_sha():
                    'rev-parse',
                    'HEAD']
 
-    return subprocess.check_output(git_command).rstrip()
+    return subprocess.check_output(
+        git_command,
+        universal_newlines=True).rstrip()
 
 
 def get_git_branch():
@@ -69,7 +71,9 @@ def get_git_branch():
                    '--abbrev-ref',
                    'HEAD']
 
-    return subprocess.check_output(git_command).rstrip()
+    return subprocess.check_output(
+        git_command,
+        universal_newlines=True).rstrip()
 
 
 def get_git_desc():
@@ -81,7 +85,9 @@ def get_git_desc():
                    '--dirty',
                    '--abbrev=40']
 
-    return subprocess.check_output(git_command).rstrip()
+    return subprocess.check_output(
+        git_command,
+        universal_newlines=True).rstrip()
 
 
 def run_packer(mmw_config, machine_types, aws_profile):

src/mmw/mmw/settings/base.py

@@ -110,6 +110,7 @@ def get_env_setting(setting):
 CELERY_BROKER_URL = 'redis://{0}:{1}/2'.format(
     environ.get('MMW_CACHE_HOST', 'localhost'),
     environ.get('MMW_CACHE_PORT', 6379))
+CELERY_BROKER_CONNECTION_RETRY_ON_STARTUP = True
 
 CELERY_IMPORTS = (
     # Submodule task is not always autodiscovered

src/mmw/requirements/base.txt

@@ -6,7 +6,7 @@ djangorestframework==3.13.1
 django-filter==21.1
 django-registration-redux==2.9
 django-extensions==3.1.5
-git+https://github.com/azavea/python-omgeo.git@6.2.1#egg=python-omgeo
+python-omgeo==6.2.2
 rauth==0.7.3
 djangorestframework-gis==0.17.0
 drf_yasg==1.20.0
@@ -15,8 +15,8 @@ cryptography==36.0.1
 pyOpenSSL==21.0.0
 markdown==3.3.6
 tr55==1.3.0
-gwlf-e==3.1.0
-requests[security]==2.26.0
+gwlf-e==3.3.0
+requests[security]==2.28.2
 rollbar==0.16.2
 retry==0.9.2
 python-dateutil==2.8.2
@@ -25,8 +25,8 @@ django_celery_results==2.2.0
 pandas==1.3.5
 ulmo==0.8.8
 hs_restclient==1.3.7
-fiona==1.8.20
+fiona==1.8.21
 redis<4
-numpy==1.20.3
+numpy==1.22.*
 dictdiffer==0.9.0
 BMPxlsx==3.0.0

src/mmw/testem.json

@@ -1,6 +1,6 @@
 {
   "launch_in_ci": [
-    "Firefox"
+    "Chromium"
   ],
   "test_page": "test.html"
 }