Release/1.30.0

.github/workflows/github-actions.yml

@@ -0,0 +1,68 @@
+on:
+  push:
+    branches: [ "release/1.30.0" ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    services:
+      postgres:
+        image: postgres:latest
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: github_actions
+        ports:
+          - 5432:5432
+        # needed because the postgres container does not provide a healthcheck
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python 3.9
+      uses: actions/setup-python@v4
+      with:
+        python-version: 3.9
+    - name: Install dependencies
+      run: |
+        pip install pip==19.3.1
+        pip install --force-reinstall 'setuptools<58.0.0'
+        pip install -r requirements.txt
+        pip install -r requirements-toil.txt
+        #python manage.py migrate
+    - name: Run migrations
+      run: python manage.py migrate
+    - name: Run test
+      #run: python manage.py test
+      run: |
+        coverage run --source='.' manage.py test
+        coverage report -m --fail-under=75
+    - name: Run flake8
+      uses: suo/flake8-github-action@releases/v1
+      with:
+          checkName: 'build'   # NOTE: this needs to be the same as the job name
+    env:
+      RIDGEBACK_DB_NAME: github_actions
+      RIDGEBACK_DB_PASSWORD: postgres
+      RIDGEBACK_DB_USERNAME: postgres
+      RIDGEBACK_TOIL_JOB_STORE_ROOT: /sample_path
+      RIDGEBACK_TOIL_WORK_DIR_ROOT: /sample_path
+      RIDGEBACK_TOIL_TMP_DIR_ROOT: /sample_path
+      RIDGEBACK_LSF_WALLTIME: 10:00
+      RIDGEBACK_LSF_SLA: SLA
+      CELERY_LOG_PATH: /sample_path
+      CELERY_PID_PATH: /sample_path
+      CELERY_BEAT_SCHEDULE_PATH: /sample_path
+      RIDGEBACK_NEXTFLOW_JOB_STORE_ROOT: /sample_path
+      RIDGEBACK_NEXTFLOW_WORK_DIR_ROOT: /sample_path
+      RIDGEBACK_NEXTFLOW_TMP_DIR_ROOT: /sample_path
+      SINGULARITY_PATH: /sample_singularity
+      RIDGEBACK_VENV: /sample_path
+      RIDGEBACK_PATH: /sample_path
+      RIDGEBACK_PORT: 4009
+      RIDGEBACK_DEFAULT_QUEUE: sample_queue
+      RIDGEBACK_RABBITMQ_USERNAME: sample_username
+      RIDGEBACK_RABBITMQ_PASSWORD: sample_password

Jenkinsfile

@@ -0,0 +1,41 @@
+pipeline {
+  agent any
+  parameters {
+      choice(name: 'SERVER', choices: ['DEV', 'STAGE', 'PROD'], description: 'Server')
+  }
+  stages {
+      stage('Deploy to Dev') {
+      when {
+        expression { params.SERVER == 'DEV' }
+      }
+      steps {
+        echo 'deply to dev'
+        sshagent(credentials: ['a4d999a5-6318-4659-83be-3f148a5490ca']) {
+            sh 'ssh  -o StrictHostKeyChecking=no -o UserKnownHostsFile=/juno/work/ci/jenkins/known_hosts [email protected] "cd /srv/services/ridgeback_dev_2/code/ridgeback && source /juno/work/ci/jenkins/lsf.sh && git pull && git checkout $BRANCH_NAME && cd /srv/services/ridgeback_dev_2 && source run_restart.sh"'
+        }
+      }
+      }
+      stage('Deploy to Stage') {
+      when {
+        expression { params.SERVER == 'STAGE' }
+      }
+      steps {
+        echo 'deply to stage'
+        sshagent(credentials: ['a4d999a5-6318-4659-83be-3f148a5490ca']) {
+          sh 'ssh  -o StrictHostKeyChecking=no -o UserKnownHostsFile=/juno/work/ci/jenkins/known_hosts [email protected] "cd /srv/services/staging_voyager/ridgeback/code/ridgeback && source /juno/work/ci/jenkins/lsf.sh && git pull && git checkout $BRANCH_NAME && cd ../.. && source run_restart.sh"'
+        }
+      }
+      }
+    stage('Deploy to Prod') {
+          when {
+          expression { params.SERVER == 'PROD' }
+          }
+      steps {
+        echo 'deply to PROD'
+        sshagent(credentials: ['a4d999a5-6318-4659-83be-3f148a5490ca']) {
+          sh 'ssh  -o StrictHostKeyChecking=no -o UserKnownHostsFile=/juno/work/ci/jenkins/known_hosts [email protected] "cd /srv/services/ridgeback/code/ridgeback && source /juno/work/ci/jenkins/lsf.sh && git pull && git checkout $BRANCH_NAME && git pull && source run_restart.sh"'
+        }
+      }
+    }
+  }
+}

container/Readme.md

@@ -17,72 +17,64 @@ Here are both the essential and optional environment variables needed by the ins
 
 ##### General
 
- Variable       | Description
- :------------- |:-------------
-SINGULARITYENV_SINGULARITY_PATH | The path to the singularity executable
-SINGULARITYENV_RIDGEBACK_VENV | The path to the venv containing toil and other python dependencies
+| Variable                        | Description                                                        |
+| :------------------------------ | :----------------------------------------------------------------- |
+| SINGULARITYENV_SINGULARITY_PATH | The path to the singularity executable                             |
+| SINGULARITYENV_RIDGEBACK_VENV   | The path to the venv containing toil and other python dependencies |
 
-Optional Variable       | Description | Default
-:------------- |:------------- |:-------------
-SINGULARITYENV_RIDGEBACK_PATH | The path to the ridgeback repo | /usr/bin/ridgeback (in container)
-SINGULARITYENV_RIDGEBACK_PORT | The port to run the ridgeback webserver | 8000
+| Optional Variable             | Description                             | Default                           |
+| :---------------------------- | :-------------------------------------- | :-------------------------------- |
+| SINGULARITYENV_RIDGEBACK_PATH | The path to the ridgeback repo          | /usr/bin/ridgeback (in container) |
+| SINGULARITYENV_RIDGEBACK_PORT | The port to run the ridgeback webserver | 8000                              |
 
 ##### Rabbitmq
 
- Variable       | Description
- :------------- |:-------------
- SINGULARITYENV_RIDGEBACK_DEFAULT_QUEUE | Default rabbitmq queue
- SINGULARITYENV_RIDGEBACK_RABBITMQ_USERNAME | rabbitmq username
- SINGULARITYENV_RIDGEBACK_RABBITMQ_PASSWORD | rabbitmq password
-
- The default rabbitmq queue should be the same queue set in toil_orchestrator/celery.py
+| Variable                                   | Description            |
+| :----------------------------------------- | :--------------------- |
+| SINGULARITYENV_RIDGEBACK_DEFAULT_QUEUE     | Default rabbitmq queue |
+| SINGULARITYENV_RIDGEBACK_RABBITMQ_USERNAME | rabbitmq username      |
+| SINGULARITYENV_RIDGEBACK_RABBITMQ_PASSWORD | rabbitmq password      |
 
+The default rabbitmq queue should be the same queue set in orchestrator/celery.py
 
 ##### Database
 
-Variable       | Description
-:------------- |:-------------
-SINGULARITYENV_RIDGEBACK_DB_NAME | Database name
-SINGULARITYENV_RIDGEBACK_DB_USERNAME | Database username
-SINGULARITYENV_RIDGEBACK_DB_PASSWORD | Database password
-SINGULARITYENV_RIDGEBACK_DB_PORT | Database port
-
+| Variable                             | Description       |
+| :----------------------------------- | :---------------- |
+| SINGULARITYENV_RIDGEBACK_DB_NAME     | Database name     |
+| SINGULARITYENV_RIDGEBACK_DB_USERNAME | Database username |
+| SINGULARITYENV_RIDGEBACK_DB_PASSWORD | Database password |
+| SINGULARITYENV_RIDGEBACK_DB_PORT     | Database port     |
 
 ##### Toil
 
-Variable       | Description
-:------------- |:-------------
-SINGULARITYENV_RIDGEBACK_TOIL_JOB_STORE_ROOT | TOIL jobstore
-SINGULARITYENV_RIDGEBACK_TOIL_WORK_DIR_ROOT | TOIL workdir
-SINGULARITYENV_RIDGEBACK_TOIL_TMP_DIR_ROOT | TOIL tmp dir
-
+| Variable                                     | Description   |
+| :------------------------------------------- | :------------ |
+| SINGULARITYENV_RIDGEBACK_TOIL_JOB_STORE_ROOT | TOIL jobstore |
+| SINGULARITYENV_RIDGEBACK_TOIL_WORK_DIR_ROOT  | TOIL workdir  |
+| SINGULARITYENV_RIDGEBACK_TOIL_TMP_DIR_ROOT   | TOIL tmp dir  |
 
 ##### LSF
 
-Variable       | Description
-:------------- |:-------------
-SINGULARITYENV_LSF_LIBDIR | The path to the lsf lib dir
-SINGULARITYENV_LSF_SERVERDIR | The path to the lsf etc dir
-SINGULARITYENV_LSF_ENVDIR | The path to the lsf.conf
-SINGULARITYENV_LSF_BINDIR | The path to the lsf bin dir
-SINGULARITYENV_RIDGEBACK_LSF_SLA | Service SLA for LSF jobs
-
-Optional Variable       | Description | Default
-:------------- |:------------- |:-------------
-SINGULARITYENV_RIDGEBACK_LSF_WALLTIME | walltime limit for LSF jobs | None
-SINGULARITYENV_RIDGEBACK_LSF_STACKLIMIT | stacklimit for LSF | None
+| Variable                         | Description                 |
+| :------------------------------- | :-------------------------- |
+| SINGULARITYENV_LSF_LIBDIR        | The path to the lsf lib dir |
+| SINGULARITYENV_LSF_SERVERDIR     | The path to the lsf etc dir |
+| SINGULARITYENV_LSF_ENVDIR        | The path to the lsf.conf    |
+| SINGULARITYENV_LSF_BINDIR        | The path to the lsf bin dir |
+| SINGULARITYENV_RIDGEBACK_LSF_SLA | Service SLA for LSF jobs    |
 
+| Optional Variable                       | Description        | Default |
+| :-------------------------------------- | :----------------- | :------ |
+| SINGULARITYENV_RIDGEBACK_LSF_STACKLIMIT | stacklimit for LSF | None    |
 
 ##### Celery
 
-Optional Variable       | Description | Default
-:--- | :--- | :---
-SINGULARITYENV_CELERY_LOG_PATH | Path to store the celery log files | /tmp
-SINGULARITYENV_CELERY_PID_PATH | Path to store the celery pid files | /tmp
-SINGULARITYENV_CELERY_BEAT_SCHEDULE_PATH | Path to store the beat schedule path | /tmp
-
-
-
+| Optional Variable                        | Description                          | Default |
+| :--------------------------------------- | :----------------------------------- | :------ |
+| SINGULARITYENV_CELERY_LOG_PATH           | Path to store the celery log files   | /tmp    |
+| SINGULARITYENV_CELERY_PID_PATH           | Path to store the celery pid files   | /tmp    |
+| SINGULARITYENV_CELERY_BEAT_SCHEDULE_PATH | Path to store the beat schedule path | /tmp    |
 
 #### Configure singularity mount points
 
@@ -95,6 +87,7 @@ export SINGULARITY_BIND="/juno,$SINGULARITYENV_LSF_LIBDIR,$SINGULARITYENV_LSF_SE
 #### Running an instance
 
 Running the following command will create a ridgeback instance named `ridgeback_service`
+
 ```
 singularity instance start ridgeback_service.sif ridgeback_service
 ```

container/ridgeback_service.def

@@ -95,16 +95,6 @@ Includecmd: no
     fi
 
     ### LSF Parameters to communicate with LSF
-    if [ -z "$RIDGEBACK_LSF_WALLTIME" ]; then
-        echo "ERROR: SINGULARITYENV_RIDGEBACK_LSF_WALLTIME not set."
-        exit 1
-    fi
-
-    if [ -z "$RIDGEBACK_LSF_SLA" ]; then
-        echo "ERROR: SINGULARITYENV_RIDGEBACK_LSF_SLA not set."
-        exit 1
-    fi
-
     if [ -z "$LSF_ENVDIR" ]; then
         echo "ERROR: SINGULARITYENV_LSF_ENVDIR is not set."
         exit 1
@@ -124,7 +114,7 @@ Includecmd: no
         echo "ERROR: SINGULARITYENV_LSF_SERVERDIR is not set."
         exit 1
     fi
-
-    python3 ${RIDGEBACK_PATH}/manage.py migrate
-    yes | python3 ${RIDGEBACK_PATH}/manage.py collectstatic
+    
+    python3 ${RIDGEBACK_PATH}/manage.py migrate --noinput
+    python3 ${RIDGEBACK_PATH}/manage.py collectstatic --noinput
     python3 ${RIDGEBACK_PATH}/manage.py runserver 0.0.0.0:$RIDGEBACK_PORT > /dev/null 2>&1 < /dev/null &

integration_tests/Jenkinsfile

@@ -0,0 +1,27 @@
+pipeline {
+  agent any
+  stages {
+      stage('Deploy to Stage') {
+        steps {
+        build job: 'ridgeback-deploy/develop', parameters: [[$class: 'StringParameterValue', name: 'SERVER', value: 'STAGE']],  propagate: true, wait: true
+        }
+      }
+      stage('Run integration tests') {
+        steps {
+          build job: 'beagle_deployment_staging', propagate: true, wait: true
+        }
+      }
+  }
+  post {
+        failure {
+        slackSend channel: '#robot-house',
+                  color: 'bad',
+                  message: "The pipeline ${currentBuild.fullDisplayName} failed."
+        }
+        success {
+        slackSend channel: '#robot-house',
+                  color: 'good',
+                  message: "The pipeline ${currentBuild.fullDisplayName} completed successfully."
+        }
+  }
+}

orchestrator/admin.py

@@ -48,7 +48,6 @@ def cleanup_files(self, request, queryset):
 Already cleaned up {cleaned_up}
         """
         for job in queryset:
-
             if all([job.job_store_clean_up, job.working_dir_clean_up]):
                 already_cleaned_up_projects = already_cleaned_up_projects + 1
             elif any([job.job_store_clean_up, job.working_dir_clean_up]):

orchestrator/commands/command.py

@@ -10,6 +10,7 @@ class CommandType(IntEnum):
     SUSPEND = 4
     RESUME = 5
     SET_OUTPUT_PERMISSION = 6
+    CHECK_HANGING = 7
 
 
 class Command(object):

orchestrator/migrations/0013_auto_20231116_1546.py

@@ -0,0 +1,23 @@
+# Generated by Django 2.2.24 on 2023-11-16 20:46
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("orchestrator", "0012_job_base_dir"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="job",
+            name="tool_walltime",
+            field=models.IntegerField(default=1440),
+        ),
+        migrations.AlterField(
+            model_name="job",
+            name="walltime",
+            field=models.IntegerField(default=7200),
+        ),
+    ]

orchestrator/models.py

@@ -15,6 +15,7 @@ def message_default():
         "log": "",
         "failed_jobs": {},
         "unknown_jobs": {},
+        "alerts": [],
         "info": "",
     }
     return message_default_dict
@@ -173,7 +174,8 @@ class Job(BaseModel):
     submitted = models.DateTimeField(blank=True, null=True)
     finished = models.DateTimeField(blank=True, null=True)
     track_cache = JSONField(blank=True, null=True)
-    walltime = models.IntegerField(default=4320)
+    walltime = models.IntegerField(default=7200)
+    tool_walltime = models.IntegerField(default=1440)
     memlimit = models.CharField(blank=True, null=True, default=None, max_length=20)
     metadata = JSONField(blank=True, null=True, default=dict)
 

orchestrator/scheduler/scheduler.py

@@ -19,7 +19,9 @@ def get_jobs_to_submit():
         LONG: 7200 <= walltime
         """
         short_jobs_count = Job.objects.filter(
-            status__gte=Status.SUBMITTING, status__lt=Status.COMPLETED, walltime__lt=settings.SHORT_JOB_MAX_DURATION
+            status__gte=Status.SUBMITTING,
+            status__lt=Status.COMPLETED,
+            walltime__lt=settings.SHORT_JOB_MAX_DURATION,
         ).count()
         medium_jobs_count = Job.objects.filter(
             status__gte=Status.SUBMITTING,
@@ -28,19 +30,21 @@ def get_jobs_to_submit():
             walltime__lt=settings.MEDIUM_JOB_MAX_DURATION,
         ).count()
         long_jobs_count = Job.objects.filter(
-            status__gte=Status.SUBMITTING, status__lt=Status.COMPLETED, walltime__gte=settings.MEDIUM_JOB_MAX_DURATION
+            status__gte=Status.SUBMITTING,
+            status__lt=Status.COMPLETED,
+            walltime__gte=settings.MEDIUM_JOB_MAX_DURATION,
         ).count()
         pending_jobs_short = Job.objects.filter(
             status__lt=Status.SUBMITTING, walltime__lt=settings.SHORT_JOB_MAX_DURATION
-        ).order_by("created_date")[: settings.SHORT_JOB_QUEUE - short_jobs_count]
+        ).order_by("created_date")[: max(settings.SHORT_JOB_QUEUE - short_jobs_count, 0)]
         pending_jobs_medium = Job.objects.filter(
             status__lt=Status.SUBMITTING,
             walltime__gte=settings.SHORT_JOB_MAX_DURATION,
             walltime__lt=settings.MEDIUM_JOB_MAX_DURATION,
-        ).order_by("created_date")[: settings.MEDIUM_JOB_QUEUE - medium_jobs_count]
+        ).order_by("created_date")[: max(settings.MEDIUM_JOB_QUEUE - medium_jobs_count, 0)]
         pending_jobs_long = Job.objects.filter(
             status__lt=Status.SUBMITTING, walltime__gte=settings.MEDIUM_JOB_MAX_DURATION
-        ).order_by("created_date")[: settings.LONG_JOB_QUEUE - long_jobs_count]
+        ).order_by("created_date")[: max(settings.LONG_JOB_QUEUE - long_jobs_count, 0)]
         jobs_to_submit = []
         jobs_to_submit.extend(pending_jobs_short)
         jobs_to_submit.extend(pending_jobs_medium)