Changes to CCM seqr instance (2024-09-19)

.dockerignore

@@ -8,3 +8,10 @@ deploy/*
 .git
 .vscode
 .idea
+
+data/
+gunicorn/
+logs/
+!logs/django/
+nginx/
+scripts/

.gitignore

@@ -18,3 +18,14 @@ pedigree_images/*
 seqr_settings
 
 django_key
+
+data/*
+
+hail-elasticsearch-pipelines
+nginx/certs/*
+!nginx/certs/.gitkeep
+pipeline-logs
+sql-dumps
+
+.vscode/
+pipeline_runner/

Dockerfile

@@ -0,0 +1 @@
+deploy/docker/seqr/Dockerfile

deploy/docker/seqr/Dockerfile

@@ -86,6 +86,7 @@ COPY --from=build /build/ui/dist /seqr/ui/dist
 ENV VIRTUAL_ENV=/opt/venv
 ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 
+COPY logs/django /var/log/django
 RUN ./manage.py collectstatic --no-input
 
 EXPOSE 8000

deploy/docker/seqr/config/gunicorn_config.py

@@ -1,6 +1,7 @@
 command = 'gunicorn'
 bind = '0.0.0.0:8000'
 workers = 1
-loglevel = 'info'
+loglevel = 'debug'
 timeout = 3600   # seconds (default is 30)
-errorlog = '-'  # logs to stderr
+accesslog = '/var/log/gunicorn/access.log'
+errorlog = '/var/log/gunicorn/error.log'

docker-compose.yml

@@ -15,7 +15,6 @@ services:
       timeout: 10s
       retries: 100
 
-
   redis:
     image: gcr.io/seqr-project/redis:gcloud-prod
     healthcheck:
@@ -24,11 +23,10 @@ services:
       timeout: 5s
       retries: 100
 
-
   elasticsearch:
     image: docker.elastic.co/elasticsearch/elasticsearch:7.16.3
     volumes:
-      - ./data/elasticsearch:/usr/share/elasticsearch/data
+      - /mnt/hpf/data/elasticsearch:/usr/share/elasticsearch/data
     container_name: elasticsearch
     environment:
       - http.host=0.0.0.0
@@ -42,28 +40,34 @@ services:
     ports:
       - 9200:9200
 
-
-  kibana:
-    image: docker.elastic.co/kibana/kibana:7.16.3
-    environment:
-      - ELASTICSEARCH_HOSTS=http://elasticsearch:9200
-    depends_on:
-      elasticsearch:
-        condition: service_healthy
-    healthcheck:
-      test: bash -c "curl -s kibana:5601 | grep kibana"
-      interval: 3s
-      timeout: 5s
-      retries: 100
+#  kibana:
+#    image: docker.elastic.co/kibana/kibana:7.16.3
+#    environment:
+#      - ELASTICSEARCH_HOSTS=http://elasticsearch:9200
+#    depends_on:
+#      elasticsearch:
+#        condition: service_healthy
+#    healthcheck:
+#      test: bash -c "curl -s kibana:5601 | grep kibana"
+#      interval: 3s
+#      timeout: 5s
+#      retries: 100
 
   seqr:
-    image: gcr.io/seqr-project/seqr:gcloud-prod
+    build: .
+    image: ccm-seqr:dev
     volumes:
-    - ./data/readviz:/readviz
-    - ./data/seqr_static_files:/seqr_static_files
+      - /mnt/hpf/bam_files:/readviz
+      # - ./data/readviz:/readviz
+      - /mnt/hpf/data/seqr_static_files:/seqr_static_files
+      - ./settings.py:/seqr/settings.py
+      - ./gunicorn/gunicorn_config.py:/seqr/gunicorn_config.py
+      - ./logs/django:/var/log/django
+      - ./logs/gunicorn:/var/log/gunicorn
     ports:
-      - 80:8000
+      - 8000:8000
     environment:
+      - BASE_URL=https://seqr.ccm.sickkids.ca
       - SEQR_GIT_BRANCH=dev
       - PYTHONPATH=/seqr
       - STATIC_MEDIA_DIR=/seqr_static_files
@@ -77,6 +81,7 @@ services:
       - PGPORT=5433
       - PGUSER=postgres
       - GUNICORN_WORKER_THREADS=4
+      - ANALYST_USER_GROUP=analyst
     depends_on:
       postgres:
         condition: service_healthy
@@ -87,15 +92,33 @@ services:
     healthcheck:
       test: bash -c "curl -s 'http://localhost:8000' | grep html"
 
-
   pipeline-runner:
-    image: gcr.io/seqr-project/pipeline-runner:gcloud-prod
+    image: gcr.io/seqr-project/pipeline-runner:20220526_130649
     volumes:
-      - ./data/seqr-reference-data:/seqr-reference-data
-      - ./data/vep_data:/vep_data
-      - ./data/input_vcfs:/input_vcfs
+      - /mnt/hpf/data/seqr-reference-data:/seqr-reference-data
+      - /mnt/hpf/data/vep_data:/vep_data
+      - /mnt/hpf/data/input_vcfs:/input_vcfs
+      - ./pipeline_runner/download_reference_data.sh:/usr/local/bin/download_reference_data.sh
+      - ./pipeline_runner/ensembl-vep-release-104:/ensembl-vep-release-104
+      - ./pipeline_runner/entrypoint.sh:/entrypoint.sh
+      - ./pipeline_runner/update_clinvar_matrix_table.sh:/usr/local/bin/update_clinvar_matrix_table.sh
+      - ./pipeline_runner/update_combined_reference_data.sh:/usr/local/bin/update_combined_reference_data.sh
+      - ./pipeline_runner/vep_configs/vep-GRCh37-loftee.json:/vep_configs/vep-GRCh37-loftee.json
+      - ./pipeline_runner/vep_configs/vep-GRCh38-loftee.json:/vep_configs/vep-GRCh38-loftee.json
       - ~/.config:/root/.config
     depends_on:
       elasticsearch:
         condition: service_healthy
 
+  nginx:
+    image: nginx
+    ports:
+      - 80:80
+      - 443:443
+    depends_on:
+      seqr:
+        condition: service_healthy
+    volumes:
+      - ./nginx/conf.d/default.conf:/etc/nginx/conf.d/default.conf
+      - ./nginx/certs:/etc/nginx/certs
+      - ./logs/nginx:/var/log/nginx

gunicorn/gunicorn_config.py

@@ -0,0 +1,7 @@
+command = 'gunicorn'
+bind = '0.0.0.0:8000'
+workers = 1
+loglevel = 'debug'
+timeout = 3600   # seconds (default is 30)
+accesslog = '/var/log/gunicorn/access.log'
+errorlog = '/var/log/gunicorn/error.log'

nginx/conf.d/default.conf

@@ -0,0 +1,38 @@
+access_log /var/log/nginx/access.log;
+error_log /var/log/nginx/error.log debug;
+
+server {
+  listen 80 default_server;
+  listen [::]:80 default_server;
+  server_name _;
+  return 301 https://$host$request_uri;
+}
+
+server {
+  listen 443 ssl http2 default_server;
+  listen [::]:443 ssl http2 default_server;
+  server_name _;
+
+  ssl_certificate /etc/nginx/certs/bundle.crt;
+  ssl_certificate_key /etc/nginx/certs/star_ccm_sickkids_ca.key;
+  ssl_session_timeout 1d;
+  ssl_session_cache shared:MozSSL:10m;  # about 40000 sessions
+  ssl_session_tickets off;
+
+  ssl_dhparam /etc/nginx/certs/dhparam.pem;
+
+  ssl_protocols TLSv1.2 TLSv1.3;
+  ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384;
+  ssl_prefer_server_ciphers off;
+
+  add_header Strict-Transport-Security "max-age=63072000" always;
+
+  ssl_stapling on;
+  ssl_stapling_verify on;
+
+  location / {
+    proxy_pass http://seqr:8000;
+    proxy_set_header Connection "";
+    proxy_read_timeout 300s;
+  }
+}

requirements.in

@@ -5,6 +5,7 @@ django-guardian                   # object-level permissions for database record
 django-hijack                     # allows admins to login as other user
 django-cors-headers               # allows CORS requests for client-side development
 django-storages[google]==1.11.1   # alternative GCS storage backend for the django media_root
+django-extensions
 social-auth-app-django            # the package for Django to authenticate users with social medieas
 social-auth-core                  # the Python social authentication package. Required by social-auth-app-django
 elasticsearch==7.9.1              # elasticsearch client

requirements.txt

@@ -84,6 +84,7 @@ django-cors-headers==3.11.0
     # via -r requirements.in
 django-csp==3.7
     # via -r requirements.in
+django-extensions==3.2.3
 django-guardian==2.4.0
     # via -r requirements.in
 django-hijack==3.1.6

run_seqr_pipeline.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+#usage: ~/ccm_seqr/run_seqr_pipeline.sh <path_to_famID_txt> <path_to_vcf_folder> <sample_type> <genome_version>
+#the VCF path would be the path after GRCh37/38 folder ex: Muise/C1012
+
+input_vcf_fam_list=$1
+vcf_folder=$2
+sample_type=$3
+genome_version=$4
+
+while read -r fam
+do
+echo $fam
+vcf=$(ls $fam*.vcf.gz)
+index=$(echo $fam | tr '[:upper:]' '[:lower:]')
+docker-compose exec pipeline-runner python3 -m seqr_loading SeqrMTToESTask --local-scheduler --reference-ht-path /seqr-reference-data/GRCh38/combined_reference_data_grch38.ht --clinvar-ht-path /seqr-reference-data/GRCh38/clinvar.GRCh38.ht --vep-config-json-path /vep_configs/vep-GRCh38-loftee.json --es-host elasticsearch --es-index-min-num-shards 1 --sample-type WES --es-index musie_japanese --genome-version 38 --source-paths /input_vcfs/GRCh38/Muise/japanese_cohort/musie_japanese.vcf.gz --dest-path /input_vcfs/GRCh38/Muise/japanese_cohort/musie_japanese.mt --dont-validate > pipeline-logs/err_file_musie_japanese-dontval.txt
+tmux new-session -d -s $fam
+tmux send-keys -t $fam ${tmux_commad} ENTER
+echo 'moving to the next'
+done < $input_vcf_fam_list

scripts/remove_contigs.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+BUILD_VERSION=GRCh$1
+VCF_DIR=./data/input_vcfs/$BUILD_VERSION
+INPUT_FILENAME=$2
+INPUT_FILE=$VCF_DIR/$INPUT_FILENAME
+SED_SCRIPT="/$3/d"
+OUTPUT_FILE=$VCF_DIR/${INPUT_FILENAME/.vcf.gz/_2.vcf.gz}
+
+zcat $INPUT_FILE | sed $SED_SCRIPT | bgzip > $OUTPUT_FILE

scripts/run_pipeline.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+BUILD_VERSION=$2
+FILE_PATH=GRCh${BUILD_VERSION}
+FILENAME=$1
+SAMPLE_TYPE=$3
+INDEX_NAME=$4
+INPUT_FILE_PATH=${FILE_PATH}/${FILENAME}
+
+docker-compose exec pipeline-runner load_data.sh $BUILD_VERSION $SAMPLE_TYPE $INDEX_NAME $INPUT_FILE_PATH

seqr/management/commands/send_reset_password.py

@@ -0,0 +1,14 @@
+from django.core.management.base import BaseCommand
+from django.contrib.auth.models import User
+
+from seqr.utils.communication_utils import send_reset_password_email
+
+
+class Command(BaseCommand):
+
+    def add_arguments(self, parser):
+        parser.add_argument('-e', '--email-address', required=True, help="Email address of an existing user in the DB.")
+
+    def handle(self, *args, **options):
+        user = User.objects.get(email__iexact=options['email_address'])
+        send_reset_password_email(user)

seqr/urls.py

@@ -124,7 +124,7 @@
 from seqr.views.apis.superuser_api import get_all_users
 
 from seqr.views.apis.awesomebar_api import awesomebar_autocomplete_handler
-from seqr.views.apis.auth_api import login_required_error, login_view, logout_view, policies_required_error
+from seqr.views.apis.auth_api import login_required_error, login_view, logout_view, policies_required_error, register_view
 from seqr.views.apis.igv_api import fetch_igv_track, receive_igv_table_handler, update_individual_igv_sample, \
     igv_genomes_proxy
 from seqr.views.apis.analysis_group_api import update_analysis_group_handler, delete_analysis_group_handler
@@ -159,6 +159,7 @@
     'matchmaker/matchbox',
     'matchmaker/disclaimer',
     'privacy_policy',
+    'register/',
     'terms_of_service',
 
 ]
@@ -273,6 +274,7 @@
     'matchmaker/update_project_contact/(?P<project_guid>[^/]+)': update_mme_project_contact,
 
     'login': login_view,
+    'register': register_view,
     'users/forgot_password': forgot_password,
     'users/(?P<username>[^/]+)/set_password': set_password,
     'users/update': update_user,

seqr/utils/communication_utils.py

@@ -1,7 +1,9 @@
 import logging
+from requests.utils import quote
 from slacker import Slacker
 from settings import SLACK_TOKEN, BASE_URL, ANVIL_UI_URL
 from django.core.mail import EmailMultiAlternatives
+from django.template.loader import render_to_string
 from django.utils.html import strip_tags
 
 from seqr.views.utils.terra_api_utils import anvil_enabled
@@ -35,7 +37,7 @@ def send_welcome_email(user, referrer):
         )
         setup_message += ' Once you are registered in AnVIL, you will be able to access seqr at {}'.format(BASE_URL)
     else:
-        setup_message = 'Please click this link to set up your account:\n    {}login/set_password/{}'.format(
+        setup_message = 'Please click this link to set up your account:\n    {}/login/set_password/{}'.format(
             BASE_URL, user.password)
 
     email_content = """
@@ -61,3 +63,15 @@ def send_html_email(email_body, **kwargs):
     )
     email_message.attach_alternative(email_body, 'text/html')
     email_message.send()
+
+
+def send_reset_password_email(user):
+    subject = 'seqr: Reset your password'
+    to = user.email
+    email_body = render_to_string('emails/reset_password.html', {
+        'base_url': BASE_URL,
+        'full_name': user.get_full_name(),
+        'password_token': quote(user.password, safe='')
+    })
+
+    send_html_email(email_body, subject=subject, to=[to])

seqr/utils/elasticsearch/utils.py

@@ -18,7 +18,7 @@ class InvalidSearchException(Exception):
     pass
 
 
-def get_es_client(timeout=60, **kwargs):
+def get_es_client(timeout=300, **kwargs):
     client_kwargs = {
         'hosts': [{'host': ELASTICSEARCH_SERVICE_HOSTNAME, 'port': ELASTICSEARCH_SERVICE_PORT}],
         'timeout': timeout,