Merge pull request #92 from croth1/bug_fixes

small bug fixes/improvements

.env_template

@@ -23,6 +23,7 @@ WEBSERVER_DIR=/var/webserver/BaMM_webserver
 MEDIA_DIR=/var/webserver/media_db
 LOG_DIR=/var/webserver/logs
 MOTIF_DB_DIR=/var/webserver/motif_db
+BACKUP_DIR=/var/webserver/backups
 
 # email logger setttings
 # ######################
@@ -113,3 +114,12 @@ MAX_JOB_STORAGE_DAYS=7
 
 # maximum number of days input files are stored on the server
 MAX_INPUT_STORAGE_DAYS=2
+
+# maximum number of backups kept
+N_STORED_BACKUPS=10
+
+# set to 2 to run the backups every day at 2 AM UTC
+DAILY_BACKUP_HOUR_UTC=2
+
+# set to 3 to run the cleanup every day at 3 AM UTC
+DAILY_CLEANUP_HOUR_UTC=3

.gitignore

@@ -1,18 +1,4 @@
-ExampleData_BaMMmotif.zip
-/bammmotif/static/scripts/*.html
-/media/
-/DB/
-/bammmotif/static/DB/*
-/bammmotif/templates/home/test.html
-/bammmotif/static/scripts/Occurrence_plots.R
-/bammmotif/static/scripts/plotly.py
-categories.json
-confusables.json
-bammmotif/templates/results/result_detail_backup.html
-bammmotif/static/img/owl_01.png
-bammmotif/static/img/owl_02.png
-
-
+celerybeat.pid
 .env
 
 # Byte-compiled / optimized / DLL files

Dockerfile

@@ -9,11 +9,12 @@ RUN apt-get update && apt-get upgrade -y && apt-get install -y \
 	libxslt-dev     \
 	libffi-dev      \
 	libssl-dev      \ 
-    libboost-all-dev\
-    cmake           \
-    imagemagick     \
-    ghostscript     \
-    build-essential 
+    	libboost-all-dev\
+    	cmake           \
+    	imagemagick     \
+    	ghostscript     \
+    	build-essential \
+	mysql-client
 
 # install python dependencies
 COPY requirements.txt /code/

README.md

@@ -20,7 +20,7 @@ All persistent data will be stored on the host. I am using the directory `/var/w
 ```bash
 WEBSERVER_DIR=/var/webserver
 # create folder structure
-mkdir -p $WEBSERVER_DIR/{media_db,logs,motif_db,mysql_db,redis_db}
+mkdir -p $WEBSERVER_DIR/{media_db,logs,motif_db,mysql_db,redis_db,backup}
 
 cd $WEBSERVER_DIR
 git clone https://github.com/soedinglab/BaMM_webserver.git

bammmotif/database/views.py

@@ -19,7 +19,7 @@ def maindb(request):
 
             search_term = form.cleaned_data['search_term']
             if not search_term:
-                form = DBForm()
+                form = DBForm(request.POST)
                 return render(request, 'database/db_main.html', {'form': form, 'warning': True})
             db_entries = ChIPseq.objects.filter(target_name__icontains=search_term,
                                                 motif_db=motif_db)
@@ -33,7 +33,7 @@ def maindb(request):
                         'is_bamm_database': motif_db.model_parameters.modelorder > 0,
                     })
             else:
-                form = DBForm()
+                form = DBForm(request.POST)
                 return render(request, 'database/db_main.html', {'form': form, 'warning': True})
     else:
         form = DBForm()

bammmotif/management/commands/cleanup.py

@@ -0,0 +1,8 @@
+from bammmotif.utils.cleanup import full_cleanup
+from django.conf import settings
+from django.core.management.base import BaseCommand
+
+
+class Command(BaseCommand):
+    def handle(self, *args, **options):
+        full_cleanup()

bammmotif/static/js/bamm_jq.js

@@ -0,0 +1,7 @@
+$(document).ready(function(){
+
+	// https://stackoverflow.com/a/18537624/2272172
+	$("#checkAll").change(function(){
+	    $('input:checkbox').not(this).prop('checked', this.checked);
+	});
+});

bammmotif/tasks.py

@@ -1,10 +1,12 @@
 from __future__ import absolute_import
 from contextlib import redirect_stdout, redirect_stderr
 import shutil
+import os
 
 from celery import task
 from django.conf import settings
 from django.utils import timezone
+from django.core import management
 
 from .commands import Compress
 from .utils.path_helpers import (
@@ -30,39 +32,10 @@ def generic_model_zip_task(job):
 
 @task
 def cleanup_task():
-    cleanup_expired_jobs()
-    cleanup_input_files()
+    management.call_command('cleanup')
 
 
-def cleanup_input_files():
-    min_time = timezone.now() - timezone.timedelta(days=settings.MAX_INPUT_STORAGE_DAYS)
-    timed_out_jobs = JobInfo.objects.filter(created_at__lt=min_time, has_input=True,
-                                            is_example=False)
-
-    for job in timed_out_jobs:
-        logger.debug('Job %s selected for input cleanup.', job.job_id)
-        input_folder = get_job_input_folder(job.job_id)
-        job.has_input = False
-        job.save()
-        try:
-            shutil.rmtree(input_folder)
-        except FileNotFoundError:
-            logger.warn('Could not find input folder %s', input_folder)
-
-    logger.info('Cleaned up input files of %s jobs', len(timed_out_jobs))
-
-
-def cleanup_expired_jobs():
-    min_time = timezone.now() - timezone.timedelta(days=settings.MAX_JOB_STORAGE_DAYS)
-    timed_out_jobs = JobInfo.objects.filter(created_at__lt=min_time, is_example=False)
-
-    for job in timed_out_jobs:
-        logger.debug('Job %s selected for removal.', job.job_id)
-        job_folder = get_job_folder(job.job_id)
-        job.delete()
-        try:
-            shutil.rmtree(job_folder)
-        except FileNotFoundError:
-            logger.warn('Could not find job folder %s', job_folder)
-
-    logger.info('Removed %s expired jobs', len(timed_out_jobs))
+@task
+def full_backup():
+    management.call_command('dbbackup', noinput=True)
+    management.call_command('mediabackup', noinput=True, compress=True)

bammmotif/templates/.gitignore

@@ -0,0 +1 @@
+home/notification.html

bammmotif/templates/peng/peng_result.html

@@ -16,70 +16,6 @@ <h2> Results for {{ job_info.job_name }} </h2>
             </div>
             {% endif %}
 
-
-        {% comment %}
-        <form name="call_bamm" method="post" action="{% url 'peng_to_bamm' pk=result.job_ID %}">
-            {% csrf_token %}
-            <div>
-                <input type="hidden" value="{% for motif in meme_meta_info %} {{ motif.select }} {% endfor %}"
-                       name="meme_meta_info"/>
-                <input type="hidden" value="{{ result.job_ID }}" name="pk">
-                <input type="submit" value="call peng_to_bamm"/>
-            </div>
-        </form>
-        <script language="JavaScript">
-        </script>
-    <script type="text/javascript" src="{{ STATIC_URL }} /static/js/jquery-3.1.1.js">
-        $("#checkAll").click(function(){
-            $('input:checkbox').not(this).prop('checked', this.checked);
-        });
-    </script>
-    <script language="javascript">
-        $("#checkAlsdfsl").change(
-            function(){
-                $("input[type=checkbox]").attr('checked', this.checked);
-            }
-        );
-    </script>
-        {% endcomment %}
-    {% comment %}
-    <script type="text/javascript">
-        function checkAll(source) {
-            checkboxes = document.getElementById('meme_ids');
-            for(var i=0; i<checkboxes.length; i++) {
-                checkboxes[i].checked = source.checked;
-            }
-        }
-    </script>
-
-    <script language="JavaScript">
-        var select_all = document.getElementById("select_all"); //select all checkbox
-        var checkboxes = document.getElementsByClassName("checkbox"); //checkbox items
-
-        //select all checkboxes
-        select_all.addEventListener("change", function(e){
-        for (i = 0; i < checkboxes.length; i++) {
-        checkboxes[i].checked = select_all.checked;
-        }
-        });
-
-
-        for (var i = 0; i < checkboxes.length; i++) {
-        checkboxes[i].addEventListener('change', function(e){ //".checkbox" change
-        //uncheck "select all", if one of the listed checkbox item is unchecked
-        if(this.checked == false){
-        select_all.checked = false;
-        }
-        //check "select all" if all checkbox items are checked
-        if(document.querySelectorAll('.checkbox:checked').length == checkboxes.length){
-        select_all.checked = true;
-        }
-        });
-        }
-        </script>
-    {% endcomment %}
-
-
         <h2>Discovered seeds</h2>
 
 	<p>You can select up to {{ max_seeds }} seeds for higher-order refinement. </p>

bammmotif/utils/cleanup.py

@@ -0,0 +1,75 @@
+import shutil
+import os
+from logging import getLogger
+
+from django.conf import settings
+from django.utils import timezone
+
+from ..models.models import JobInfo
+from .path_helpers import (
+    get_job_input_folder,
+    get_job_folder,
+)
+
+
+logger = getLogger(__name__)
+
+
+def cleanup_input_files():
+    min_time = timezone.now() - timezone.timedelta(days=settings.MAX_INPUT_STORAGE_DAYS)
+    timed_out_jobs = JobInfo.objects.filter(created_at__lt=min_time, has_input=True,
+                                            is_example=False)
+
+    for job in timed_out_jobs:
+        logger.debug('Job %s selected for input cleanup.', job.job_id)
+        input_folder = get_job_input_folder(job.job_id)
+        job.has_input = False
+        job.save()
+        try:
+            shutil.rmtree(input_folder)
+        except FileNotFoundError:
+            logger.warn('Could not find input folder %s', input_folder)
+
+    logger.info('removed input files of %s jobs', len(timed_out_jobs))
+
+
+def cleanup_expired_jobs():
+    min_time = timezone.now() - timezone.timedelta(days=settings.MAX_JOB_STORAGE_DAYS)
+    timed_out_jobs = JobInfo.objects.filter(created_at__lt=min_time, is_example=False)
+
+    for job in timed_out_jobs:
+        logger.debug('job %s selected for removal.', job.job_id)
+        job_folder = get_job_folder(job.job_id)
+        job.delete()
+        try:
+            shutil.rmtree(job_folder)
+            logger.debug('removing expired job folder: %s' % job_folder)
+        except FileNotFoundError:
+            logger.warn('could not find job folder %s', job_folder)
+        except OSError as exc:
+            logger.error(exc)
+
+    logger.info('removed %s expired jobs', len(timed_out_jobs))
+
+
+def remove_orphan_jobdirs():
+    job_ids = {str(job.job_id) for job in JobInfo.objects.all()}
+    n_orphans = 0
+    for job_dir in os.listdir(settings.JOB_DIR):
+        if job_dir not in job_ids:
+            try:
+                full_job_dir_path = get_job_folder(job_dir)
+                if not os.path.isdir(full_job_dir_path):
+                    continue
+                shutil.rmtree(full_job_dir_path)
+                logger.debug('removing orphaned job folder: %s', full_job_dir_path)
+                n_orphans += 1
+            except OSError as exc:
+                logger.error(exc)
+    logger.info('removed %s orphaned job folders', n_orphans)
+
+
+def full_cleanup():
+    cleanup_expired_jobs()
+    cleanup_input_files()
+    remove_orphan_jobdirs()

bammmotif/utils/meme_reader.py

@@ -59,7 +59,7 @@ def meme_drop_unselected_motifs(src_path, dest_path, selected_motifs):
     with open(src_path, "r") as f:
         fcont = f.read().split("\n\n")
         meme_header = "\n\n".join(fcont[:3])
-        remaining_motifs = meme_header + "\n\n"
+        remaining_motifs = meme_header
         for elem in fcont:
             if not elem.startswith("MOTIF"):
                 continue

bammmotif/utils/misc.py

@@ -61,6 +61,7 @@ def __exit__(self, error_type, error, tb):
 
             job.meta_job.save()
             job.save()
+
             # swallow the exception
             return True
 
@@ -71,6 +72,9 @@ def __exit__(self, error_type, error, tb):
             traceback.print_exception(error_type, error, tb, file=sys.stdout)
             print(timezone.now(), "\t | WARNING: \t %s " % job.meta_job.status)
 
+            # swallow the exception
+            return True
+
         else:
             job.meta_job.status = self.success_status
             self.had_exception = False

docker-compose.yml

@@ -20,22 +20,14 @@ services:
 
   celery:
     build: .
-    # memo: this cannot be outsourced to a script because of two reasons:
-    # 1) sending a sigterm to a script will not send it to the worker
-    # 2) using a script with init system will attempt to kill the workers instead of a warm shutdown
-    command: celery 
-                --loglevel=${BAMM_LOG_LEVEL}
-                --app=webserver.celery:app worker
-                --soft-time-limit ${JOB_TIME_LIMIT}
-                --concurrency $N_PARALLEL_JOBS
-                -B
-                -s /logs/celerybeat-schedule
+    command: dumb-init --single-child bash ./run_workers.sh
     user: ${BAMM_USER_UID}
     volumes:
       - ${WEBSERVER_DIR}:/code
       - ${MEDIA_DIR}:/code/media
       - ${MOTIF_DB_DIR}:/motif_db
       - ${LOG_DIR}:/logs
+      - ${BACKUP_DIR}:/backup
     links:
       - redis_celery
       - db
@@ -73,6 +65,9 @@ services:
         - FDR_CV_FOLD=${FDR_CV_FOLD}
         - MAX_INPUT_STORAGE_DAYS=${MAX_INPUT_STORAGE_DAYS}
         - MAX_JOB_STORAGE_DAYS=${MAX_JOB_STORAGE_DAYS}
+        - N_STORED_BACKUPS=${N_STORED_BACKUPS}
+        - DAILY_BACKUP_HOUR_UTC=${DAILY_BACKUP_HOUR_UTC}
+        - DAILY_CLEANUP_HOUR_UTC=${DAILY_CLEANUP_HOUR_UTC}
 
   web:
     build: .
@@ -85,6 +80,7 @@ services:
       - ${MEDIA_DIR}:/code/media
       - ${MOTIF_DB_DIR}:/motif_db
       - ${LOG_DIR}:/logs
+      - ${BACKUP_DIR}:/backup
     ports:
       - ${SERVER_HOST_PORT}:10080
     links:

requirements.txt

@@ -3,6 +3,7 @@ django-secure
 django-registration
 django-ipware
 django-widget-tweaks
+django-dbbackup
 mysqlclient
 psycopg2
 Celery