Merge pull request #68 from NHS-NGS/development

Development
NHS-NGS · Sep 14, 2018 · cd9b836 · cd9b836
2 parents 23953c5 + 5e9aaa0
commit cd9b836
Show file tree

Hide file tree

Showing 14 changed files with 364 additions and 48 deletions.
diff --git a/gelweb/gel2clin/templates/gel2clin/cancer_proband.html b/gelweb/gel2clin/templates/gel2clin/cancer_proband.html
@@ -1,4 +1,4 @@
-{% extends 'gel2mdt/base.html' %}
+{% extends 'gel2clin/base.html' %}
 {% load bootstrap3 %}
 {% load static %}
 {% block content %}

diff --git a/gelweb/gel2clin/templates/gel2clin/raredisease_proband.html b/gelweb/gel2clin/templates/gel2clin/raredisease_proband.html
@@ -19,7 +19,7 @@
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
 -->
-{% extends 'gel2mdt/base.html' %}
+{% extends 'gel2clin/base.html' %}
 {% load bootstrap3 %}
 {% load static %}
 {% block content %}

diff --git a/gelweb/gel2mdt/database_utils/case_handler.py b/gelweb/gel2mdt/database_utils/case_handler.py
@@ -33,6 +33,7 @@
 import re
 import copy
 import pprint
+from tqdm import tqdm
 
 
 class Case(object):
@@ -736,7 +737,6 @@ def get_panels(self):
                 if not polled:
                     panel_file = os.path.join(panelapp_storage, '{}_{}.json'.format(panel['panelName'],
                                                                                         panel['panelVersion']))
-                    print(panel["panelName"], panel["panelVersion"])
                     if os.path.isfile(panel_file):
 
                         try:
@@ -830,9 +830,10 @@ def get_genes(self):
                 })
                 self.case.gene_manager.add_searched(transcript.gene_ensembl_id, str(transcript.gene_hgnc_id))
 
-        for gene in gene_list:
+        for gene in tqdm(gene_list, desc=self.case.request_id):
             gene['HGNC_ID'] = None
             if gene['EnsembleGeneIds']:
+                tqdm.write(gene["EnsembleGeneIds"])
                 polled = self.case.gene_manager.fetch_searched(gene['EnsembleGeneIds'])
                 if polled == 'Not_found':
                     gene['HGNC_ID'] = None

diff --git a/gelweb/gel2mdt/database_utils/multiple_case_adder.py b/gelweb/gel2mdt/database_utils/multiple_case_adder.py
@@ -33,6 +33,7 @@
 import logging
 import time
 from .demographics_handler import DemographicsHandler
+from tqdm import tqdm
 
 # set up logging
 logger = logging.getLogger(__name__)
@@ -55,7 +56,7 @@ def __init__(self, sample_type, head=None, test_data=False, skip_demographics=Fa
         :param sample: If you want to add a single sample, set this the GELID
         :param pullt3: Boolean to pull t3 variants
         """
-        logger.info("Initialising a MultipleCaseAdder.")
+        print("Initialising a MultipleCaseAdder.")
 
         if sample_type == "cancer" or sample_type == "raredisease":
             pass
@@ -82,11 +83,11 @@ def __init__(self, sample_type, head=None, test_data=False, skip_demographics=Fa
         self.sample_type = sample_type
 
         if self.test_data:
-            logger.info("Fetching test data.")
+            print("Fetching test data.")
             # set list_of_cases to test zoo
             self.list_of_cases = self.fetch_test_data()
             self.cases_to_poll = None
-            logger.info("Fetched test data.")
+            print("Fetched test data.")
             self.cases_to_add = self.check_cases_to_add()
             self.cases_to_update = self.check_cases_to_update()  #
             self.cases_to_skip = set(self.list_of_cases) - \
@@ -101,41 +102,72 @@ def __init__(self, sample_type, head=None, test_data=False, skip_demographics=Fa
             self.cases_to_skip = []
         else:
             # set list_of_cases to cases of interest from API
-            logger.info("Fetching live API data.")
-            logger.info("Polling for list of available cases...")
+            print("Fetching live API data.")
+            print("Polling for list of available cases...")
             interpretation_list_poll = InterpretationList(sample_type=sample_type)
-            logger.info("Fetched available cases")
+            cases_fetched = len(interpretation_list_poll.cases_to_poll)
+            print("Fetched", cases_fetched, "available cases")
 
-            logger.info("Determining which cases to poll...")
-            self.cases_to_poll = interpretation_list_poll.cases_to_poll
-            if head:
-                self.cases_to_poll = self.cases_to_poll[:head]
-
-            logger.info("Fetching API JSON data for cases to poll...")
-            self.list_of_cases = self.fetch_api_data()
+            print("Determining which cases to poll...")
+            # reverse update, do the newest first!
+            self.total_cases_to_poll = interpretation_list_poll.cases_to_poll[::-1]
             if head:
-                # take a certain number of cases off the top
-                self.list_of_cases = self.list_of_cases[:head]
-
-            logger.info("Fetched all required CIP API data.")
-
-            logger.info("Checking which cases to add.")
-            self.cases_to_add = self.check_cases_to_add()
-            logger.info("Checking which cases require updating.")
-            self.cases_to_update = self.check_cases_to_update()#
-            self.cases_to_skip = set(self.list_of_cases) - \
-                set(self.cases_to_add) - \
-                set(self.cases_to_update)
+                self.total_cases_to_poll = self.total_cases_to_poll[:head]
+            self.num_cases_to_poll = len(self.total_cases_to_poll)
+
+            # work out how many hundreds in fetched cases
+            num_full_bins = self.num_cases_to_poll // 100
+            num_bins = num_full_bins + 1
+            final_bin_size = self.num_cases_to_poll - (num_full_bins * 100)
+
+            bins = []
+            for i in range(num_full_bins):
+                bins.append(
+                    [
+                        (100*i),
+                        (100*i) + 100
+                    ]
+                )   # [0, 100], [101, 200] etc
+            bins.append([num_full_bins * 100, None])
+
+            bin_count = 1
+            for b in bins:
+                print("Fetching cases", b[0], "to", b[1], "(bin", bin_count, "of", str(len(bins)) + ")")
+                self.cases_to_poll = self.total_cases_to_poll[b[0]: b[1]]
+
+                print("Fetching API JSON data for cases to poll...")
+                self.list_of_cases = self.fetch_api_data()
+                if head:
+                    # take a certain number of cases off the top
+                    self.list_of_cases = self.list_of_cases[:head]
+                print("Fetched all required CIP API data.")
+                print("Checking which cases to add.")
+                self.cases_to_add = self.check_cases_to_add()
+                print("Checking which cases require updating.")
+                self.cases_to_update = self.check_cases_to_update()#
+                self.cases_to_skip = set(self.list_of_cases) - \
+                    set(self.cases_to_add) - \
+                    set(self.cases_to_update)
+                self.update_database()
+
+                print("Finished processing bin", bin_count, "of", len(bins))
+
+                for case in self.cases_to_add:
+                    del case
+                for case in self.cases_to_update:
+                    del case
+
+                bin_count += 1
 
     def update_database(self):
         # begin update process
         # --------------------
         error = None
         try:
             logger.info("Adding cases from cases_to_add.")
-            print("Adding cases")
+            print("Adding", len(self.cases_to_add), "cases")
             self.add_cases()
-            print("Updating cases")
+            print("Updating", len(self.cases_to_update), "cases")
             self.add_cases(update=True)
             success = True
         except Exception as e:
@@ -182,18 +214,22 @@ def fetch_test_data(self):
         return list_of_cases
 
     def fetch_api_data(self):
-        list_of_cases = [
-            # list comprehension, calling self.get_case_json each time for poll
-            Case(
+        list_of_cases = []
+        # list comprehension, calling self.get_case_json each time for poll
+        for case in tqdm(self.cases_to_poll):
+            tqdm.write("Polling for: {case}".format(case=case["interpretation_request_id"]))
+
+            c = Case(
                 # instatiate a new case with the polled json
                 case_json=self.get_case_json(case["interpretation_request_id"]),
                 panel_manager=self.panel_manager,
                 variant_manager=self.variant_manager,
                 gene_manager=self.gene_manager,
                 skip_demographics=self.skip_demographics,
                 pullt3=self.pullt3
-            ) for case in self.cases_to_poll
-        ]
+            )
+            list_of_cases.append(c)
+
         print("Successfully fetched", len(list_of_cases), "cases from CIP API.")
         return list_of_cases
 
@@ -204,8 +240,6 @@ def get_case_json(self, interpretation_request_id):
         :param interpretation_request_id: an IR ID of the format XXXX-X
         :returns: A case json associated with the given IR ID from CIP-API
         """
-        logger.info("Polling API for case", interpretation_request_id)
-        print("Polling API for case", interpretation_request_id)
         request_poll = PollAPI(
             # instantiate a poll of CIP API for a given case json
             "cip_api", "interpretation-request/{id}/{version}".format(
@@ -361,8 +395,9 @@ def add_cases(self, update=False):
                 model_objects = model_type.objects.all().values(*lookups)
             elif not lookups:
                 model_objects = model_type.objects.all()
-            for case in cases:
+            for case in tqdm(cases, desc="Parsing {model_type} into DB".format(model_type=model_type.__name__)):
                 # create a CaseAttributeManager for the case
+                tqdm.write(case.request_id)
                 case.attribute_managers[model_type] = CaseAttributeManager(
                     case, model_type, model_objects)
                 # use thea attribute manager to set the case models
@@ -377,10 +412,11 @@ def add_cases(self, update=False):
                 ]
             elif many:
                 model_list = []
-                for case in cases:
+                for case in tqdm(cases, desc="Parsing {model_type} into DB".format(model_type=model_type.__name__)):
+                    tqdm.write(case.request_id)
                     attribute_manager = case.attribute_managers[model_type]
                     many_case_model = attribute_manager.case_model
-                    for case_model in many_case_model.case_models:
+                    for case_model in tqdm(many_case_model.case_models, desc=case.request_id):
                         model_list.append(case_model)
 
             # now create the required new Model instances from CaseModel lists

diff --git a/gelweb/gel2mdt/forms.py b/gelweb/gel2mdt/forms.py
@@ -376,3 +376,7 @@ class GenomicsEnglandform(forms.Form):
     # Version number of the interpretation
     ir_version = forms.IntegerField(label='Version')
     report_version = forms.IntegerField(label='Clinical Report Version')
+
+
+class GeneSearchForm(forms.Form):
+    gene = forms.CharField(max_length=25, widget = forms.TextInput(attrs={'style': 'width:200px'}))
diff --git a/gelweb/gel2mdt/management/commands/run_batch_update.py b/gelweb/gel2mdt/management/commands/run_batch_update.py
@@ -54,4 +54,3 @@ def handle(self, *args, **options):
                                 test_data=options['test_data'],
                                 skip_demographics=options['skip_demographics'],
                                 pullt3=options['pullt3'])
-        mca.update_database()
diff --git a/gelweb/gel2mdt/static/css/bokeh-0.12.15.min.css b/gelweb/gel2mdt/static/css/bokeh-0.12.15.min.css
diff --git a/gelweb/gel2mdt/static/js/bokeh-0.12.15.min.js b/gelweb/gel2mdt/static/js/bokeh-0.12.15.min.js
diff --git a/gelweb/gel2mdt/templates/gel2mdt/audit.html b/gelweb/gel2mdt/templates/gel2mdt/audit.html
@@ -1,8 +1,9 @@
 {% extends 'gel2mdt/base.html' %}
+{% load static %}
 {% block bokeh_head %}
 <meta charset="UTF-8">
-<script src="http://cdn.pydata.org/bokeh/release/bokeh-0.12.15.min.js"></script>
-<link rel="stylesheet" href="http://cdn.pydata.org/bokeh/release/bokeh-0.12.15.min.css">
+<script src="{% static 'js/bokeh-0.12.15.min.js' %}"></script>
+<link rel="stylesheet" href="{% static 'css/bokeh-0.12.15.min.css' %}">
 {% if script %}
 {{script|safe}}
 {% endif %}

diff --git a/gelweb/gel2mdt/templates/gel2mdt/cancer_main.html b/gelweb/gel2mdt/templates/gel2mdt/cancer_main.html
@@ -181,7 +181,7 @@ <h4 name="filter-spec" class="panel-title">
                                             </td>
                                         </tr>
 
-                                        <tr>
+                                        <tr hidden>
                                             <td>Trio status:</td>
                                             <td>
                                                 <div class="btn-group" data-toggle="buttons">
@@ -201,7 +201,7 @@ <h4 name="filter-spec" class="panel-title">
                                             </td>
                                         </tr>
 
-                                        <tr>
+                                        <tr hidden>
                                             <td><em>De novo</em> status:</td>
                                             <td>
                                                 <div class="btn-group" data-toggle="buttons">
@@ -220,6 +220,19 @@ <h4 name="filter-spec" class="panel-title">
                                     </tbody>
                                 </table>
                                 </form>
+                                <form action="{% url 'gene_search' sample_type=sample_type %}" method="post">
+                                    {% csrf_token %}
+                                 <table class="table">
+                                    <tbody>
+                                        <tr>
+                                            <td width="12.2%">Gene Search:</td>
+                                            <td width="13%">{{ gene_search_form.gene }}</td> <td>
+                        <button type="submit" ><i class="fas fa-search " style="font-size:20px;"></i></button>
+                                        </td>
+                                        </tr>
+                                    </tbody>
+                                 </table>
+                                </form>
                             </div>
                         </div>
                     </div>