Merge pull request #3934 from broadinstitute/dev

Dev

.github/workflows/hail-search-unit-tests.yaml

@@ -17,7 +17,7 @@ on:
 jobs:
   hail_search:
     runs-on: ubuntu-latest
-    container: hailgenetics/hail:0.2.126
+    container: hailgenetics/hail:0.2.128
 
     steps:
       - uses: actions/checkout@v2

CHANGELOG.md

@@ -2,6 +2,9 @@
 
 ## dev
 
+## 3/1/24
+* Add subscribable project notifications (REQUIRES DB MIGRATION)
+
 ## 1/8/24
 * Support OMIM entries with no associated gene and remove phenotypic_series_number (REQUIRES DB MIGRATION)
 

deploy/kubectl_helpers/logs.sh

@@ -6,4 +6,10 @@ set -x -e
 
 POD_NAME=$("${DIR}"/utils/get_pod_name.sh "$@")
 
-kubectl logs -f "${POD_NAME}" --all-containers
+COMPONENT=$2
+case ${COMPONENT} in
+  hail-search) CONTAINER="-c seqr-hail-search-pod" ;;
+  *) CONTAINER="--all-containers" ;;
+esac
+
+kubectl logs -f "${POD_NAME}" "${CONTAINER}"

deploy/kubectl_helpers/port_forward.sh

@@ -7,15 +7,9 @@ set -x -e
 COMPONENT=$2
 
 case ${COMPONENT} in
-  elasticsearch)
-    PORT=9200
-    NAME='service/elasticsearch-es-http'
-    OPEN_BROWSER=true
-    ;;
-  kibana)
-    PORT=5601
-    NAME='service/kibana-kb-http'
-    OPEN_BROWSER=true
+  hail-search)
+    PORT=5000
+    NAME='service/seqr-hail-search'
     ;;
   redis)
     PORT=6379

deploy/kubectl_helpers/set_env.sh

@@ -10,11 +10,9 @@ DEPLOYMENT_TARGET=$1
 case ${DEPLOYMENT_TARGET} in
   dev)
     CLUSTER_NAME=seqr-cluster-dev
-    NAMESPACE=gcloud-dev
     ;;
   prod)
     CLUSTER_NAME=seqr-cluster-prod
-    NAMESPACE=default
     ;;
   *)
     echo "Invalid deployment target '${DEPLOYMENT_TARGET}'"
@@ -26,4 +24,4 @@ gcloud config set core/project ${GCLOUD_PROJECT}
 gcloud config set compute/zone ${GCLOUD_ZONE}
 gcloud container clusters get-credentials --zone=${GCLOUD_ZONE} ${CLUSTER_NAME}
 
-kubectl config set-context $(kubectl config current-context) --namespace=${NAMESPACE}
+kubectl config set-context $(kubectl config current-context) --namespace=default

hail_search/deploy/Dockerfile

@@ -1,4 +1,4 @@
-FROM hailgenetics/hail:0.2.126
+FROM hailgenetics/hail:0.2.128
 
 LABEL maintainer="Broad TGG"
 

hail_search/queries/base.py

@@ -217,7 +217,7 @@ def __init__(self, sample_data, sort=XPOS, sort_metadata=None, num_results=100,
         self._has_secondary_annotations = False
         self._is_multi_data_type_comp_het = False
         self.max_unaffected_samples = None
-        self._load_table_kwargs = {}
+        self._load_table_kwargs = {'_n_partitions': (os.cpu_count() or 2)-1}
         self.entry_samples_by_family_guid = {}
 
         if sample_data:
@@ -276,6 +276,7 @@ def _parse_sample_data(self, sample_data):
     def _load_filtered_project_hts(self, project_samples, skip_all_missing=False, **kwargs):
         filtered_project_hts = []
         exception_messages = set()
+        num_projects = len(project_samples)
         for i, (project_guid, project_sample_data) in enumerate(project_samples.items()):
             project_ht = self._read_table(
                 f'projects/{project_guid}.ht',
@@ -286,7 +287,7 @@ def _load_filtered_project_hts(self, project_samples, skip_all_missing=False, **
                 continue
             try:
                 filtered_project_hts.append(
-                    (*self._filter_entries_table(project_ht, project_sample_data, **kwargs), len(project_sample_data))
+                    (*self._filter_entries_table(project_ht, project_sample_data, num_projects=num_projects, **kwargs), len(project_sample_data))
                 )
             except HTTPBadRequest as e:
                 exception_messages.add(e.reason)
@@ -344,7 +345,7 @@ def _add_project_ht(self, families_ht, project_ht, default, default_1=None):
         if default_1 is None:
             default_1 = default
 
-        if not project_ht.any(hl.is_defined(project_ht.family_entries)):
+        if not project_ht.any(project_ht.family_entries.any(hl.is_defined)):
             return families_ht, False
 
         families_ht = families_ht.join(project_ht, how='outer')
@@ -361,8 +362,7 @@ def _add_project_ht(self, families_ht, project_ht, default, default_1=None):
         ), True
 
     def _filter_entries_table(self, ht, sample_data, inheritance_filter=None, quality_filter=None, **kwargs):
-        if not self._load_table_kwargs:
-            ht = self._prefilter_entries_table(ht, **kwargs)
+        ht = self._prefilter_entries_table(ht, **kwargs)
 
         ht, sorted_family_sample_data = self._add_entry_sample_families(ht, sample_data)
 
@@ -1048,6 +1048,7 @@ def gene_counts(self):
     def lookup_variant(self, variant_id, sample_data=None):
         self._parse_intervals(intervals=None, variant_ids=[variant_id], variant_keys=[variant_id])
         ht = self._read_table('annotations.ht', drop_globals=['paths', 'versions'])
+        self._load_table_kwargs['_n_partitions'] = 1
         ht = ht.filter(hl.is_defined(ht[XPOS]))
 
         annotation_fields = self.annotation_fields(include_genotype_overrides=False)

hail_search/queries/snv_indel.py

@@ -73,12 +73,13 @@ class SnvIndelHailTableQuery(MitoHailTableQuery):
         PATHOGENICTY_HGMD_SORT_KEY: lambda r: [MitoHailTableQuery.CLINVAR_SORT(CLINVAR_KEY, r), r.hgmd.class_id],
     }
 
-    def _prefilter_entries_table(self, ht, *args, **kwargs):
+    def _prefilter_entries_table(self, ht, *args, num_projects=1, **kwargs):
         ht = super()._prefilter_entries_table(ht, *args, **kwargs)
-        af_ht = self._get_loaded_filter_ht(
-            GNOMAD_GENOMES_FIELD, 'high_af_variants.ht', self._get_gnomad_af_prefilter, **kwargs)
-        if af_ht:
-            ht = ht.filter(hl.is_missing(af_ht[ht.key]))
+        if num_projects > 1 or not self._load_table_kwargs.get('_filter_intervals'):
+            af_ht = self._get_loaded_filter_ht(
+                GNOMAD_GENOMES_FIELD, 'high_af_variants.ht', self._get_gnomad_af_prefilter, **kwargs)
+            if af_ht:
+                ht = ht.filter(hl.is_missing(af_ht[ht.key]))
         return ht
 
     def _get_gnomad_af_prefilter(self, frequencies=None, pathogenicity=None, **kwargs):

requirements.in

@@ -3,6 +3,7 @@ django-anymail                    # for sending emails using cloud-based mail se
 django-csp                        # for setting CSP headers
 django-guardian                   # object-level permissions for database records. Behind a major version due to missing Python 2 support
 django-hijack                     # allows admins to login as other user
+django-notifications-hq           # notification app
 django-cors-headers               # allows CORS requests for client-side development
 django-storages[google]==1.11.1   # alternative GCS storage backend for the django media_root
 social-auth-app-django            # the package for Django to authenticate users with social medieas

requirements.txt

@@ -34,7 +34,10 @@ django==3.2.24
     #   django-csp
     #   django-guardian
     #   django-hijack
+    #   django-model-utils
+    #   django-notifications-hq
     #   django-storages
+    #   jsonfield
 django-anymail==9.0
     # via -r requirements.in
 django-cors-headers==3.13.0
@@ -45,10 +48,12 @@ django-guardian==2.4.0
     # via -r requirements.in
 django-hijack==3.2.6
     # via -r requirements.in
+django-model-utils==4.3.1
+    # via django-notifications-hq
+django-notifications-hq==1.8.3
+    # via -r requirements.in
 django-storages[google]==1.11.1
-    # via
-    #   -r requirements.in
-    #   django-storages
+    # via -r requirements.in
 elasticsearch==7.9.1
     # via
     #   -r requirements.in
@@ -84,6 +89,8 @@ idna==3.4
     # via requests
 jmespath==1.0.1
     # via -r requirements.in
+jsonfield==3.1.0
+    # via django-notifications-hq
 markdownify==0.11.6
     # via -r requirements.in
 oauthlib==3.2.2
@@ -117,7 +124,9 @@ python-dateutil==2.8.2
 python3-openid==3.2.0
     # via social-auth-core
 pytz==2022.7.1
-    # via django
+    # via
+    #   django
+    #   django-notifications-hq
 redis==4.5.4
     # via -r requirements.in
 requests==2.31.0
@@ -158,6 +167,8 @@ soupsieve==2.5
     # via beautifulsoup4
 sqlparse==0.4.4
     # via django
+swapper==1.3.0
+    # via django-notifications-hq
 tqdm==4.64.1
     # via -r requirements.in
 urllib3==1.26.18

seqr/fixtures/1kg_project.json

@@ -10,6 +10,7 @@
         "name": "1kg project n\u00e5me with uni\u00e7\u00f8de",
         "description": "1000 genomes project description with uni\u00e7\u00f8de",
         "consent_code": "H",
+        "subscribers": 6,
 	"can_edit_group": 2,
 	"can_view_group": 3,
     "workspace_name": "anvil-1kg project n\u00e5me with uni\u00e7\u00f8de",
@@ -35,6 +36,7 @@
         "consent_code": "H",
         "workspace_name": "empty",
         "workspace_namespace": "my-seqr-billing",
+        "subscribers": 6,
 	"can_edit_group": 2,
 	"can_view_group": 3,
 	"is_mme_enabled": false,
@@ -52,6 +54,7 @@
         "last_modified_date": "2017-03-13T09:07:49.582Z",
         "name": "Test Reprocessed Project",
         "description": "",
+        "subscribers": 6,
 	"can_edit_group": 2,
 	"can_view_group": 3,
     "workspace_name": "anvil-project 1000 Genomes Demo",
@@ -72,6 +75,7 @@
         "created_by": 11,
         "last_modified_date": "2017-03-13T09:07:49.582Z",
         "name": "Non-Analyst Project",
+        "subscribers": 7,
 	    "can_edit_group": 2,
 	    "can_view_group": 2,
 	    "last_accessed_date": "2017-09-15T18:15:50.827Z",
@@ -2723,5 +2727,67 @@
         "institution": "Broad Center for Mendelian Genomics",
         "patient_id": "NA19675"
     }
-}
-]
+}, {
+    "model": "notifications.notification",
+    "pk": 1,
+    "fields": {
+        "level": "info",
+        "recipient": 11,
+        "unread": true,
+        "actor_content_type": 40,
+        "actor_object_id": "1",
+        "verb": "Loaded 2 new WES SV samples",
+        "description": null,
+        "target_content_type": null,
+        "target_object_id": null,
+        "action_object_content_type": null,
+        "action_object_object_id": null,
+        "timestamp": "2023-12-17T22:14:44.568Z",
+        "public": true,
+        "deleted": false,
+        "emailed": false,
+        "data": null
+    }
+}, {
+    "model": "notifications.notification",
+    "pk": 2,
+    "fields": {
+        "level": "info",
+        "recipient": 11,
+        "unread": false,
+        "actor_content_type": 10,
+        "actor_object_id": "1",
+        "verb": "Loaded 8 new WES samples",
+        "description": null,
+        "target_content_type": null,
+        "target_object_id": null,
+        "action_object_content_type": null,
+        "action_object_object_id": null,
+        "timestamp": "2023-08-27T22:14:44.568Z",
+        "public": true,
+        "deleted": false,
+        "emailed": false,
+        "data": null
+    }
+}, {
+    "model": "notifications.notification",
+    "pk": 3,
+    "fields": {
+        "level": "info",
+        "recipient": 12,
+        "unread": true,
+        "actor_content_type": 10,
+        "actor_object_id": "1",
+        "verb": "Loaded 8 new WES samples",
+        "description": null,
+        "target_content_type": null,
+        "target_object_id": null,
+        "action_object_content_type": null,
+        "action_object_object_id": null,
+        "timestamp": "2023-08-27T22:14:44.568Z",
+        "public": true,
+        "deleted": false,
+        "emailed": false,
+        "data": null
+    }
+}]

seqr/fixtures/users.json

@@ -31,6 +31,22 @@
         "permissions": []
     }
 },
+{
+    "model": "auth.group",
+    "pk": 6,
+    "fields": {
+        "name": "subscribers",
+        "permissions": []
+    }
+},
+{
+    "model": "auth.group",
+    "pk": 7,
+    "fields": {
+        "name": "subscribers_external",
+        "permissions": []
+    }
+},
 {
     "model": "auth.user",
     "pk": 10,
@@ -62,7 +78,7 @@
         "is_staff": false,
         "is_active": true,
         "date_joined": "2017-03-12T23:09:54.180Z",
-        "groups": [],
+        "groups": [6],
         "user_permissions": []
     }
 }, {
@@ -79,7 +95,7 @@
         "is_staff": false,
         "is_active": true,
         "date_joined": "2017-03-12T23:09:54.180Z",
-        "groups": [],
+        "groups": [7],
         "user_permissions": []
     }
 }, {