Two db updates (#19)

* Update system wide requirements, removing python 2 packages and upgrading those to python 3
Updating python source version and python vars to use python 3
Update to using python -venv instead of python-virtualenv for python 3 compatibility
Update to use custom Dockerfile to install ca-certs to docker container
Update postgresql extensions for new KPI DB

* Removing deprecated CORS Model setting. See https://github.com/adamchainz/django-cors-headers/blob/master/src/corsheaders/checks.py#L153

* Add a playbook to prepare the hosts

* Update Molecule tests to run Ubuntu 18.04
Update postgresql and postgis ext versions

.travis.yml

@@ -6,7 +6,7 @@ services:
 before_install:
   - sudo apt-get -qq update
 install:
-  - pip install molecule
+  - pip install molecule==3.0.8
   - pip install testinfra
   - pip install docker-py
 script:

defaults/main.yml

@@ -12,12 +12,11 @@ kpi_system_requirements:
   - libxml2-dev
   - libxslt1-dev
   - python-celery
-  - python-pip
-  - python-psycopg2
-  - python-setuptools
+  - python3-pip
+  - python3-psycopg2
+  - python3-setuptools
   - software-properties-common
-  - python-virtualenv
-  - python2.7-dev
+  - virtualenv
   - ttfautohint
   - zlib1g-dev
   - binutils
@@ -26,8 +25,6 @@ kpi_system_requirements:
 kpi_npm_requirements:
   - webpack
 kpi_django_setup_commands:
-  - syncdb --noinput
-  - migrate --noinput
   - collectstatic --noinput
   - compilemessages
 kpi_pip_paths: "{{ kpi_checkout_path }}/dependencies/pip/external_services.txt"
@@ -43,12 +40,12 @@ kpi_git_key_ssh_file:
 kpi_git_key_filename: id_ed25519
 kpi_copy_key_from_file: false
 kpi_remove_git_key: true
-kpi_python_source_version: 2.7
-kpi_python_version: "python2.7"
+kpi_python_source_version: "3"
+kpi_python_version: python{{ kpi_python_source_version }}
 kpi_checkout_path: "{{ kpi_versioned_path }}/{{ kpi_git_branch }}"
 kpi_app_path: "{{ kpi_system_user_home }}/app"
 kpi_versioned_path: "{{ kpi_app_path }}-versioned"
-kpi_git_branch: ona-custom-changes
+kpi_git_branch: kpi_2.0
 kpi_service_name: "{{ kpi_system_user }}"
 kpi_venv_path: "{{ kpi_system_user_home }}/.virtualenvs/{{ kpi_service_name }}"
 kpi_nvm_install_path: "{{ kpi_system_user_home }}/.nvm"
@@ -69,24 +66,40 @@ kpi_python_packages:
 
 # Use the exact anonymous_user_id as the one set on the onadata/formhub/kobocat app.
 kpi_anonymous_user_id: "-1"
-kpi_pgsql_db: "onadata"
+kpi_pgsql_db: "kpi"
 kpi_pgsql_user: "kpi"
-kpi_pgsql_password: "random_password"
+kpi_pgsql_password: "kpi"
 kpi_pgsql_host: "localhost"
 kpi_pgsql_port: 5432
+ona_pgsql_db: "ona"
+ona_pgsql_user: "onadata"
+ona_pgsql_password: "onadata"
+ona_pgsql_host: "localhost"
+ona_pgsql_port: 5432
 kpi_google_analytics_token: "random_string"
 kpi_kobocat_url: "http://localhost:8002"
 kpi_enketo_url: "http://localhost:8005"
 kpi_enketo_api_token: "qwertyuiop"
-kpi_broker_url: "amqp://{{ kpi_system_user }}:{{ kpi_system_user }}@localhost:5672"
+kpi_broker_url: 'redis://localhost:6379/1'
 kpi_log_path: "/var/log/{{ kpi_service_name }}"
-kpi_cors_origin_whitelist: []
 
 # TODO: Add documentation for this config
-kpi_frontend_domain: "locahost:3001"
+kpi_frontend_domain: "http://locahost:3001"
 kpi_frontend_login_url: "http://{{ kpi_frontend_domain }}/login"
-kpi_cors_whitelist:
+kpi_cors_origin_whitelist:
   - "{{ kpi_frontend_domain }}"
+  - 'http://localhost:3000'
+  - 'http://localhost:3001'
+  - 'http://localhost:4000'
+  - 'http://0.0.0.0:3000'
+  - 'http://127.0.0.1:3000'
+  - 'http://127.0.0.1:3001'
+  - 'http://127.0.0.1:4000'
+  - 'http://localhost:8000'
+  - 'http://localhost:8080'
+  - 'http://127.0.0.1:8080'
+  - 'http://localhost:8005'
+
 kpi_wsgi_module: kobo.wsgi:application
 kpi_wsgi_processes: 12
 kpi_wsgi_env: "HTTPS=on"

meta/main.yml

@@ -4,11 +4,11 @@ galaxy_info:
   description: Installs and configures KPI Formbuilder
   company: Ona Systems Inc
   license: Apache-2.0
-  min_ansible_version: 2.6
+  min_ansible_version: 2.8
   platforms:
     - name: Ubuntu
       versions:
-        - 16.04
+        - 18.04
   galaxy_tags:
     - kpi
     - formbuilder

molecule/default/converge.yml

@@ -2,37 +2,10 @@
 - name: Converge
   hosts: all
   roles:
-    - role: onaio.postgresql
-      postgresql_backup_enabled: false
-      postgresql_version: 9.6
-      postgresql_users:
-          - name: kpi
-            pass: kpi
-            encrypted: true
-      postgresql_databases:
-        - name: kpi
-          owner: kpi
-          encoding: "UTF-8"
-          hstore: true
-          gis: true
-      postgresql_database_extensions:
-        - db: kpi
-          extensions:
-            - postgis
-      postgresql_ext_install_postgis: true
-      postgresql_ext_postgis_version: "2.3"
-      postgresql_ext_postgis_deps:
-        - libgeos-c1v5
-        - "postgresql-{{ postgresql_version }}-postgis-{{ postgresql_ext_postgis_version }}"
-        - "postgresql-{{ postgresql_version }}-postgis-{{ postgresql_ext_postgis_version }}-scripts"
-      postgresql_enable_ssl: true
-      postgresql_ssl_domain: "db.example.com"
-      postgresql_ssl_ca_key: "{{ lookup('file', 'ssl/root.key') }}"
-      postgresql_ssl_ca_cert: "{{ lookup('file', 'ssl/root.crt') }}"
     - role: ansible-kpi
       kpi_pgsql_db: kpi
       kpi_pgsql_user: kpi
       kpi_pgsql_password: kpi
       kpi_enable_monitoring: false
       kpi_git_url: "https://github.com/onaio/kpi.git"
-      kpi_git_branch: "ona-custom-changes"
+      kpi_git_branch: "kpi_2.0"

molecule/default/molecule.yml

@@ -6,8 +6,9 @@ driver:
 lint: |
   yamllint .
 platforms:
-  - name: ubuntu-16.04
-    image: solita/ubuntu-systemd:16.04
+  - name: kpi-ubuntu-18.04
+    image: solita/ubuntu-systemd:18.04
+    dockerfile: Dockerfile.j2
     privileged: true
     command: /sbin/init
 scenario:

molecule/default/prepare.yml

@@ -0,0 +1,12 @@
+
+---
+- name: Prepare
+  hosts: all
+  tasks:
+    - name: Make sure system dependencies are installed
+      apt:
+        pkg:
+          - gpg
+          - python3-psycopg2
+        state: present
+      when: ansible_lsb.major_release|int > 16

molecule/default/tests/vagranttest.yml

@@ -2,12 +2,6 @@
 - name: Pre-install stuff
   hosts: all
   tasks:
-    - name: Install python 2 if not there
-      raw: |
-        test -e /usr/bin/python3 || \
-        (apt -y update && apt install -y  python-minimal)
-      register: output
-      changed_when: output.stdout|trim() != ""
     - name: Install sudo if not there
       raw: |
         test -e /usr/bin/sudo || apt install -y sudo
@@ -20,10 +14,10 @@
   hosts: all
   vars:
     # postgres
-    kpi_anonymous_user_id: "-1"
-    kpi_pgsql_db: "onadata"
+    kpi_anonymous_user_id: "1"
+    kpi_pgsql_db: "kpi"
     kpi_pgsql_user: "kpi"
-    kpi_pgsql_password: "random_password"
+    kpi_pgsql_password: "kpi"
     kpi_pgsql_host: "localhost"
     kpi_pgsql_port: 5432
     # django

tasks/main.yml

@@ -30,12 +30,7 @@
   notify:
     - restart_service
 
-- name: Patch libgeos.py file for updated regex
-  patch:
-    src: templates/django_virtualenv_dir/site-packages/django/contrib/gis/geos/libgeos.py
-    dest: "{{ kpi_venv_path }}/lib/python2.7/site-packages/django/contrib/gis/geos/libgeos.py"
-
-- name: syncdb, migrate and collect static files
+- name: Migrate and collect static files
   django_manage:
     command: "{{ item }}"
     app_path: "{{ kpi_checkout_path }}"

templates/kpi/config.es6.j2

@@ -1,11 +1,11 @@
-const KPI_LOGIN_URL = "{{ kpi_frontend_login_url }}";
+const ZEBRA_LOGIN_URL = "{{ kpi_frontend_login_url }}";
 
 export function getAuthUrl(url) {
-  return KPI_LOGIN_URL + "?return_url=" + url;
+  return ZEBRA_LOGIN_URL + "?return_url=" + url;
 }
 
 export function getZebraLoginUrl() {
-  return KPI_LOGIN_URL;
+  return ZEBRA_LOGIN_URL;
 }
 
 const SENTY_DSN = "{{ kpi_raven_js_dsn }}";