Merge pull request #654 from upconsulting/develop

Python3 (#653)

.ebextensions/01-isiscb.config

@@ -1,9 +1,9 @@
 option_settings:
-  "aws:rds:dbinstance":
-    DBEngineVersion: 9.4.7
+  #"aws:rds:dbinstance":
+  #  DBEngineVersion: 9.4.7
   "aws:elasticbeanstalk:application:environment":
     DJANGO_SETTINGS_MODULE: "isiscb.production_settings"
-    PYTHONPATH: "/opt/python/current/app/isiscb:$PYTHONPATH"
+    #PYTHONPATH: "/opt/python/current/app/isiscb:$PYTHONPATH"
 
   "aws:elasticbeanstalk:container:python":
     WSGIPath: "isiscb/isiscb/wsgi.py"
@@ -35,81 +35,53 @@ Resources:
               DefaultValue: "aws-elasticbeanstalk-ec2-role"
 
 files:
-  /etc/httpd/conf.d/ssl.conf:
-    mode: "000644"
+
+  /etc/nginx/nginx.conf:
+    mode: "000400"
     owner: root
     group: root
     content: |
-      LoadModule wsgi_module modules/mod_wsgi.so
-      LoadModule rewrite_module modules/mod_rewrite.so
-      WSGIPythonHome /opt/python/run/venv
-      WSGISocketPrefix run/wsgi
-      WSGIRestrictEmbedded On
-      Listen 443
-      <VirtualHost *:443>
-        RequestHeader set X-Forwarded-Proto "https"
-        SSLEngine on
-        SSLCertificateFile "/etc/pki/tls/certs/server.crt"
-        SSLCertificateKeyFile "/etc/pki/tls/certs/server.key"
+      #Adjusted Elastic Beanstalk Nginx Configuration File
 
-        Alias /static/ /opt/python/current/app/static/
-        <Directory /opt/python/current/app/static>
-        Order allow,deny
-        Allow from all
-        </Directory>
+      user                    nginx;
+      error_log               /var/log/nginx/error.log warn;
+      pid                     /var/run/nginx.pid;
+      worker_processes        auto;
+      worker_rlimit_nofile    32145;
 
-        WSGIScriptAlias / /opt/python/current/app/isiscb/isiscb/wsgi.py
+      events {
+        worker_connections  1024;
+      }
 
-        <Directory /opt/python/current/app>
-        Require all granted
-        </Directory>
+      http {
+        include       /etc/nginx/mime.types;
+        default_type  application/octet-stream;
 
-        WSGIDaemonProcess wsgi-ssl processes=1 threads=15 display-name=%{GROUP} \
-          python-path=/opt/python/run/venv:/opt/python/current/app:/opt/python/run/venv/lib64/python2.7/site-packages \
-          home=/opt/python/current/app \
-          user=wsgi \
-          group=wsgi
-        WSGIProcessGroup wsgi-ssl
+        log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                          '$status $body_bytes_sent "$http_referer" '
+                          '"$http_user_agent" "$http_x_forwarded_for"';
 
-      </VirtualHost>
+        include       conf.d/*.conf;
 
-  /etc/pki/tls/certs/server.crt:
-    mode: "000400"
-    owner: root
-    group: root
-    content: |
-      -----BEGIN CERTIFICATE-----
-      MIID6DCCAtACCQCW0G8hMdcb0zANBgkqhkiG9w0BAQUFADCBtTELMAkGA1UEBhMC
-      VVMxETAPBgNVBAgTCE9rbGFob21hMQ8wDQYDVQQHEwZOb3JtYW4xNDAyBgNVBAoT
-      K0lzaXMgQmlibGlvZ3JhcGh5IG9mIHRoZSBIaXN0b3J5IG9mIFNjaWVuY2UxLDAq
-      BgNVBAMTI2lzaXNjYi1zdGFnaW5nLmVsYXN0aWNiZWFuc3RhbGsuY29tMR4wHAYJ
-      KoZIhvcNAQkBFg9zcHdlbGRvbkBvdS5lZHUwHhcNMTYwNjE1MjI0NzQwWhcNMTcw
-      NjE1MjI0NzQwWjCBtTELMAkGA1UEBhMCVVMxETAPBgNVBAgTCE9rbGFob21hMQ8w
-      DQYDVQQHEwZOb3JtYW4xNDAyBgNVBAoTK0lzaXMgQmlibGlvZ3JhcGh5IG9mIHRo
-      ZSBIaXN0b3J5IG9mIFNjaWVuY2UxLDAqBgNVBAMTI2lzaXNjYi1zdGFnaW5nLmVs
-      YXN0aWNiZWFuc3RhbGsuY29tMR4wHAYJKoZIhvcNAQkBFg9zcHdlbGRvbkBvdS5l
-      ZHUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC2KTqC8XIbpc1I5Wu6
-      rzy/FEWor5LqN/c+EgQbFfrk3XDRCTx6PHkM/mFfKEbK0sMiQK2MAzFu8FnZ5nbh
-      i0fEytuAPNd4XK2BmFEIn5svoSUzq1yo5ykj2yQVULRRsq2wTLB/+LU/QHf5QgB4
-      d7KUA3Ex7yHwegRDZdgK751Vq5HlPFgQMxg3++x5B+frHEaGMDOpVWU0yutgtbcU
-      jsUyG2GU7c+QBdkkVRqbbTtaRhd5j3e7VxyjIPXWT4PSg08KNnK3FzYqQX2WubNU
-      wC3EW/JSCXkjfY6AfgldQ9NXS+0f/g3T68c4w81MWoqDwM9+/gQACoBN6BvRVAg8
-      9nJRAgMBAAEwDQYJKoZIhvcNAQEFBQADggEBAB/Imzla1YcGiIgOoQ+2Nok3KLi7
-      O5sSMbIY+KVqfiZ2DqCnRe7m9umRaXHa/3MubLKnG9nTzNEd2caBfZl7JAtxjNTf
-      d/JhznjhhV+zsj6R/5oJX1ZfZAoDlK2RmeXSbaXja0hK1BQidIRBS0mYRy8Fxo/d
-      PgPNukhUqN90zKrIsJRyVtRhBn2iKjjvZU3MfMwpimKwuZ4z+Tyiblt8M65EqZxG
-      t2CJjYpv7GKwhcCmjGDeA2WqBVfRNih1sEl9jzwFFybGeAyTLOxSxlk7ZWAb0yql
-      ybjUGtXoPLhjHk2i3f5ThRE0FkzD+QJoMS1sh0sDxJi2VXqsXtW00PjxtF0=
-      -----END CERTIFICATE-----
+        map $http_upgrade $connection_upgrade {
+            default     "upgrade";
+        }
 
-  /etc/pki/tls/certs/server.key:
-    mode: "000400"
-    owner: root
-    group: root
-    authentication: "S3Auth"
-    source: https://s3-us-west-2.amazonaws.com/elasticbeanstalk-us-west-2-655333410217/ssl/privatekey.pem
+        server {
+            listen 80;
+            server_name _;
+            if ($http_x_forwarded_proto = 'http'){
+              return 301 https://$host$request_uri;
+            }
+
+            # Include the Elastic Beanstalk generated locations
+            include conf.d/elasticbeanstalk/*.conf;
+        }
+      }
 
 container_commands:
+  00_update_pip:
+    command: "/bin/bash .ebextensions/scripts/07_update_pip.sh"
   01_migrate:
     command: "/bin/bash .ebextensions/scripts/01_migrate.sh"
     leader_only: true
@@ -121,15 +93,7 @@ container_commands:
     leader_only: true
   04_wsgipass:
     command: "/bin/bash .ebextensions/scripts/04_wsgipass.sh"
-  05_uninstall_pil:
-    command: "/bin/bash .ebextensions/scripts/05_uninstall_pil.sh"
-  06_reinstall_pil:
-    command: "/bin/bash .ebextensions/scripts/06_reinstall_pil.sh"
-  07_update_pip:
-    command: "/bin/bash .ebextensions/scripts/07_update_pip.sh"
-  08_update_six:
-    command: "/bin/bash .ebextensions/scripts/08_update_six.sh"
-  09_update_setuptools:
-    command: "/bin/bash .ebextensions/scripts/09_update_setuptools.sh"
-  10_install_pycurl_with_nss:
-    command: "/bin/bash .ebextensions/scripts/10_install_pycurl_with_nss.sh"
+  05_create_log_file:
+      command: touch /var/log/app_analyzer.log
+  06_change_owner:
+      command: chown webapp:webapp /var/log/app_analyzer.log

.ebextensions/02-celery.config

@@ -5,82 +5,3 @@ files:
     group: root
     content: |
       celery-worker log file
-
-  "/opt/python/bin/celery_start":
-    mode: "000755"
-    owner: root
-    group: root
-    content: |
-      #!/bin/bash
-
-      NAME="isiscb"
-      #DJANGODIR=/opt/python/current/app/isiscb             # Django project directory
-
-      # Activate the virtual environment
-      #cd $DJANGODIR
-      #source /opt/python/current/env
-
-      # Start your Django Unicorn
-      # Programs meant to be run under supervisor should not daemonize themselves (do not use --daemon)
-      # Celery 4.1.0 seemed to ahve fixed the issue but created a new one where celery looses connection to the
-      # broker, so it was reverted back to 4.0.2
-      # exec /opt/python/run/venv/bin/celery worker -A isiscb --loglevel=INFO -f /opt/python/log/celery-worker.log -E -P solo
-
-  "/opt/elasticbeanstalk/hooks/appdeploy/post/run_supervised_celeryd.sh":
-    mode: "000755"
-    owner: root
-    group: root
-    content: |
-      #!/usr/bin/env bash
-
-      # ISISCB-1044: run celery as docker in extra container
-
-      # Get django environment variables
-      #celeryenv=`cat /opt/python/current/env | tr '\n' ',' | sed 's/export //g' | sed 's/$PATH/%(ENV_PATH)s/g' | sed 's/$PYTHONPATH//g' | sed 's/$LD_LIBRARY_PATH//g'`
-      #celeryenv=${celeryenv%?}
-
-      # Create celery configuraiton script
-      #celeryconf="[program:celeryd]
-      #; Set full path to celery program if using virtualenv
-      #command=/opt/python/bin/celery_start
-
-      #directory=/opt/python/current/app
-      #user=nobody
-      #numprocs=1
-      #stdout_logfile=/var/log/celery-worker.log
-      #stderr_logfile=/var/log/celery-worker.log
-      #autostart=true
-      #autorestart=true
-      #startsecs=10
-
-      #; Need to wait for currently executing tasks to finish at shutdown.
-      #; Increase this if you have very long running tasks.
-      #stopwaitsecs = 600
-
-      #; When resorting to send SIGKILL to the program to terminate it
-      #; send SIGKILL to its whole process group instead,
-      #; taking care of its children as well.
-      #killasgroup=true
-
-      #; if rabbitmq is supervised, set its priority higher
-      #; so it starts first
-      #priority=998"
-
-      # Create the celery supervisord conf script
-      #echo "$celeryconf" | tee /opt/python/etc/celery.conf
-
-      # Add configuration script to supervisord conf (if not there already)
-      #if ! grep -Fxq "[include]" /opt/python/etc/supervisord.conf
-      #    then
-      #    echo "[include]" | tee -a /opt/python/etc/supervisord.conf
-      #    echo "files: celery.conf" | tee -a /opt/python/etc/supervisord.conf
-      #fi
-
-      # Reread the supervisord config
-      #/usr/local/bin/supervisorctl -c /opt/python/etc/supervisord.conf reread
-
-      # Update supervisord in cache without restarting all services
-      #/usr/local/bin/supervisorctl -c /opt/python/etc/supervisord.conf update
-
-      # Start/Restart celeryd through supervisord
-      #/usr/local/bin/supervisorctl -c /opt/python/etc/supervisord.conf restart celeryd

.ebextensions/packages.config

@@ -1,12 +1,10 @@
 packages:
   yum:
     git: []
-    postgresql93-devel: []
     freetype-devel: []
     libpng-devel: []
     zlib-devel: []
     libjpeg-turbo-devel: []
-    mod24_ssl : []
     openssl-devel: []
     libcurl: []
     libcurl-devel: []

.ebextensions/scripts/00_install_pkgs.sh

@@ -0,0 +1,5 @@
+echo 'Installing packages'
+
+amazon-linux-extras enable postgresql11
+yum clean metadata
+yum install postgresql11-devel

.ebextensions/scripts/01_migrate.sh

@@ -4,10 +4,13 @@ echo 'This is a docker environment: $IS_DOCKER'
 
 if [ "$IS_DOCKER" != "true" ]; then
   echo 'Migrating...'
-  source /opt/python/run/venv/bin/activate
+  #source /opt/python/run/venv/bin/activate
+  source /var/app/venv/staging-LQM1lest/bin/activate
+  pip install -r requirements.txt
   cd isiscb
   python manage.py migrate auth --noinput
   python manage.py migrate --noinput
+  python manage.py createcachetable
 else
   echo 'The script 01_migrate.sh is only executed in non-docker environments.'
 fi

.ebextensions/scripts/02_collectstatic.sh

@@ -4,7 +4,8 @@ echo 'This is a docker environment: $IS_DOCKER'
 
 if [ "$IS_DOCKER" != "true" ]; then
   echo 'Collecting statics...'
-  source /opt/python/run/venv/bin/activate
+  #source /opt/python/run/venv/bin/activate
+  source /var/app/venv/staging-LQM1lest/bin/activate
   cd isiscb
   python manage.py collectstatic --noinput
 else

.ebextensions/scripts/03_createsu.sh

@@ -4,7 +4,8 @@ echo 'This is a docker environment: $IS_DOCKER'
 
 if [ "$IS_DOCKER" != "true" ]; then
   echo 'Creating superuser...'
-  source /opt/python/run/venv/bin/activate
+  #source /opt/python/run/venv/bin/activate
+  source /var/app/venv/staging-LQM1lest/bin/activate
   cd isiscb
   python manage.py createsu
 else

.ebextensions/scripts/07_update_pip.sh

@@ -4,7 +4,8 @@ echo 'This is a docker environment: $IS_DOCKER'
 
 if [ "$IS_DOCKER" != "true" ]; then
   echo 'Updating pip...'
-  source /opt/python/run/venv/bin/activate
+  #source /opt/python/run/venv/bin/activate
+  source /var/app/venv/staging-LQM1lest/bin/activate
   pip install -U pip
 else
   echo 'The script 07_update_pip.sh is only executed in non-docker environments.'

.travis.yml

@@ -1,8 +1,9 @@
 language: python
 python:
-- '2.7'
+- '3.8'
 services:
 - postgresql
+- elasticsearch
 env:
   global:
     - DJANGO_SETTINGS_MODULE=isiscb.test_settings
@@ -12,16 +13,18 @@ env:
     - SOCIAL_AUTH_TWITTER_SECRET=''
     - BOTO_CONFIG=/dev/null
 before_script:
-- psql -c 'create database isiscb;' -U postgres
+- psql -c 'create database test_isiscb;' -U postgres
+- sleep 10
 before_install:
 - pip install -U pip
 - pip install -U six
-- pip install -U setuptools==33.1.1
+- pip install -U setuptools
 - sudo apt-get install libgnutls28-dev
 script:
 - pip install nose
 - cd isiscb
 - git clone http://github.com/ubernostrum/django-registration.git --depth 2
+- python manage.py migrate
 - python -Wi manage.py test
 - cd ..
 deploy:
@@ -31,41 +34,41 @@ deploy:
       secure: "LtRgo+nPOgpOzAj2WBTJm5LhymiSOlkUQQ4xoLpo0lbUxXAJR64ma0VK31XuPDzT8wAW9K5UpELZVI7uyeh/iz/8Al5cV53TCuKlEYeKhSauoCfEexRdCQp0qe9/F9DjOVPU+yO3e/ADou6dc7e7SyvS9zalafRn/rms8Qko6iiu88Bi1JFggCYiubisyzHPfT8bLhE/pB9efxkgiP5m8kbuXLzTKBfK5nAP4W0LcA7ATA0Urk9FF+fT/fZoZsDAHvF62i6h5zxoi7+FR9dSCITTuJBsyYvLErhlAB5vlEL5deAvJvJUKv1rrhPWfoO2PW++BWNdE+DMUqDP6FVd7sAgMXRM/yqZO5n6DFYQJQHjyPRK2HLDHjIPFqfeokvq8JnuBcXpa3ajln8+Zz2YlCl0ZPLFj1q5hHGiwhp3+BwG/LTzza9heMlPDwdD5MdOXNpclzw/xEp/ffE/boriDyq2wrDRKOseeoY8ts3yByLqhqigzufei4Yj/4CxnEX9SZnF2O+nPzaMih6mf5+KTqTM6s5GENNR//w233WS62lNxF2rJhvNES3i3OCzb+cVwY5kQgELiLWgUb4hLOy1NJttVHFFB/xLATG3nditXq5TN2XFEkcVt2nlUOyCd5Nay0aNH2Qbo2iGhcBBjCetdjgmqBd/fv7PTo+YRWZWkx4="
     region: us-west-2
     app: IsisCB
-    env: isiscb-staging-2
+    env: isiscb-production-p3-3
     bucket_name: elasticbeanstalk-us-west-2-655333410217
     on:
       repo: upconsulting/IsisCB
-      branch: develop
+      tags: true
   - provider: elasticbeanstalk
     access_key_id: AKIAI3SRBWU3O3X5S7CQ
     secret_access_key:
       secure: "LtRgo+nPOgpOzAj2WBTJm5LhymiSOlkUQQ4xoLpo0lbUxXAJR64ma0VK31XuPDzT8wAW9K5UpELZVI7uyeh/iz/8Al5cV53TCuKlEYeKhSauoCfEexRdCQp0qe9/F9DjOVPU+yO3e/ADou6dc7e7SyvS9zalafRn/rms8Qko6iiu88Bi1JFggCYiubisyzHPfT8bLhE/pB9efxkgiP5m8kbuXLzTKBfK5nAP4W0LcA7ATA0Urk9FF+fT/fZoZsDAHvF62i6h5zxoi7+FR9dSCITTuJBsyYvLErhlAB5vlEL5deAvJvJUKv1rrhPWfoO2PW++BWNdE+DMUqDP6FVd7sAgMXRM/yqZO5n6DFYQJQHjyPRK2HLDHjIPFqfeokvq8JnuBcXpa3ajln8+Zz2YlCl0ZPLFj1q5hHGiwhp3+BwG/LTzza9heMlPDwdD5MdOXNpclzw/xEp/ffE/boriDyq2wrDRKOseeoY8ts3yByLqhqigzufei4Yj/4CxnEX9SZnF2O+nPzaMih6mf5+KTqTM6s5GENNR//w233WS62lNxF2rJhvNES3i3OCzb+cVwY5kQgELiLWgUb4hLOy1NJttVHFFB/xLATG3nditXq5TN2XFEkcVt2nlUOyCd5Nay0aNH2Qbo2iGhcBBjCetdjgmqBd/fv7PTo+YRWZWkx4="
     region: us-west-2
     app: IsisCB
-    env: celery-docker-staging
+    env: isiscb-production-celery-p3
     bucket_name: elasticbeanstalk-us-west-2-655333410217
     on:
       repo: upconsulting/IsisCB
-      branch: develop
+      tags: true
   - provider: elasticbeanstalk
     access_key_id: AKIAI3SRBWU3O3X5S7CQ
     secret_access_key:
       secure: "LtRgo+nPOgpOzAj2WBTJm5LhymiSOlkUQQ4xoLpo0lbUxXAJR64ma0VK31XuPDzT8wAW9K5UpELZVI7uyeh/iz/8Al5cV53TCuKlEYeKhSauoCfEexRdCQp0qe9/F9DjOVPU+yO3e/ADou6dc7e7SyvS9zalafRn/rms8Qko6iiu88Bi1JFggCYiubisyzHPfT8bLhE/pB9efxkgiP5m8kbuXLzTKBfK5nAP4W0LcA7ATA0Urk9FF+fT/fZoZsDAHvF62i6h5zxoi7+FR9dSCITTuJBsyYvLErhlAB5vlEL5deAvJvJUKv1rrhPWfoO2PW++BWNdE+DMUqDP6FVd7sAgMXRM/yqZO5n6DFYQJQHjyPRK2HLDHjIPFqfeokvq8JnuBcXpa3ajln8+Zz2YlCl0ZPLFj1q5hHGiwhp3+BwG/LTzza9heMlPDwdD5MdOXNpclzw/xEp/ffE/boriDyq2wrDRKOseeoY8ts3yByLqhqigzufei4Yj/4CxnEX9SZnF2O+nPzaMih6mf5+KTqTM6s5GENNR//w233WS62lNxF2rJhvNES3i3OCzb+cVwY5kQgELiLWgUb4hLOy1NJttVHFFB/xLATG3nditXq5TN2XFEkcVt2nlUOyCd5Nay0aNH2Qbo2iGhcBBjCetdjgmqBd/fv7PTo+YRWZWkx4="
     region: us-west-2
-    app: IsisCB
-    env: isiscb-production-6
+    app: IsisCB Staging Python 3
+    env: isiscb-staging-p3-2
     bucket_name: elasticbeanstalk-us-west-2-655333410217
     on:
       repo: upconsulting/IsisCB
-      tags: true
+      branch: develop
   - provider: elasticbeanstalk
     access_key_id: AKIAI3SRBWU3O3X5S7CQ
     secret_access_key:
       secure: "LtRgo+nPOgpOzAj2WBTJm5LhymiSOlkUQQ4xoLpo0lbUxXAJR64ma0VK31XuPDzT8wAW9K5UpELZVI7uyeh/iz/8Al5cV53TCuKlEYeKhSauoCfEexRdCQp0qe9/F9DjOVPU+yO3e/ADou6dc7e7SyvS9zalafRn/rms8Qko6iiu88Bi1JFggCYiubisyzHPfT8bLhE/pB9efxkgiP5m8kbuXLzTKBfK5nAP4W0LcA7ATA0Urk9FF+fT/fZoZsDAHvF62i6h5zxoi7+FR9dSCITTuJBsyYvLErhlAB5vlEL5deAvJvJUKv1rrhPWfoO2PW++BWNdE+DMUqDP6FVd7sAgMXRM/yqZO5n6DFYQJQHjyPRK2HLDHjIPFqfeokvq8JnuBcXpa3ajln8+Zz2YlCl0ZPLFj1q5hHGiwhp3+BwG/LTzza9heMlPDwdD5MdOXNpclzw/xEp/ffE/boriDyq2wrDRKOseeoY8ts3yByLqhqigzufei4Yj/4CxnEX9SZnF2O+nPzaMih6mf5+KTqTM6s5GENNR//w233WS62lNxF2rJhvNES3i3OCzb+cVwY5kQgELiLWgUb4hLOy1NJttVHFFB/xLATG3nditXq5TN2XFEkcVt2nlUOyCd5Nay0aNH2Qbo2iGhcBBjCetdjgmqBd/fv7PTo+YRWZWkx4="
     region: us-west-2
-    app: IsisCB
-    env: celery-docker-production-2
+    app: IsisCB Celery Staging Python 3 (non worker)
+    env: isiscb-celery-staging-p3-webinst
     bucket_name: elasticbeanstalk-us-west-2-655333410217
     on:
       repo: upconsulting/IsisCB
-      tags: true
+      branch: develop

Dockerfile

@@ -1,10 +1,14 @@
-FROM python:2.7-onbuild
+FROM python:3.6-buster
 
 ENV DJANGO_SETTINGS_MODULE=isiscb.production_settings
 
 RUN mkdir -p /var/logs/ && touch /var/logs/celery-worker.log
 
-EXPOSE 80
+COPY isiscb/ ./isiscb
+COPY requirements.txt .
+RUN pip install -r requirements.txt
 
+# Docker/AWS requests this although we don't need it?
+EXPOSE 80
 WORKDIR isiscb
 CMD ["celery", "worker", "-A", "isiscb", "--loglevel=INFO", "-f", "/var/logs/celery-worker.log", "-E", "-P", "solo"]