Skip to content

Commit

Permalink
Merge pull request #87 from NHS-NGS/development
Browse files Browse the repository at this point in the history 
Development
  • Loading branch information
@theocole
theocole authored Oct 3, 2018
2 parents 69bb8ce + 60a2c82 commit 939ac04
Show file tree
Hide file tree
Showing 18 changed files with 1,010 additions and 485 deletions.
6 changes: 0 additions & 6 deletions gelweb/gel2clin/templates/gel2clin/base.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,12 +99,6 @@
<ul class="nav navbar-nav">
<li><a href="{% url 'recent-mdt' sample_type=sample_type %}" ><i class="fas fa-history"></i> Recent MDTs</a></li>
</ul>
<ul class="nav navbar-nav">
<li><a href="{% url 'audit' sample_type=sample_type %}" ><i class="fas fa-tachometer-alt"></i> Audit</a></li>
</ul>
<ul class="nav navbar-nav">
<li><a href="{% url 'validation-list' sample_type=sample_type %}" ><i class="fas fa-check-circle"></i> Validation</a></li>
</ul>
{% endif %}


Expand Down
2 changes: 2 additions & 0 deletions gelweb/gel2clin/templates/gel2clin/raredisease_proband.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,7 @@ <h1>{{report.ir_family.participant_family.proband.gel_id}}</h1>
<th>MDT Date</th>
<th>Status</th>
<th>Creator</th>
<th>Outcome Form</th>

</tr>
</thead>
Expand All @@ -190,6 +191,7 @@ <h1>{{report.ir_family.participant_family.proband.gel_id}}</h1>
<td>{{mdt.MDT.date_of_mdt|date}}</td>
<td>{{mdt.MDT.get_status_display}}</td>
<td>{{mdt.MDT.creator}}</td>
<td><a href="/export_mdt_outcome_form/{{report.id}}"><i class="fas fa-external-link-alt"></i> MDM Outcome</a></td>

</tr>
{% endfor %}
Expand Down
100 changes: 67 additions & 33 deletions gelweb/gel2mdt/database_utils/multiple_case_adder.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,9 @@ class MultipleCaseAdder(object):
required related instances to the database and reporting status and
errors during the process.
"""
def __init__(self, sample_type, head=None, test_data=False, skip_demographics=False, sample=None, pullt3=True):
def __init__(self, sample_type, head=None, test_data=False,
skip_demographics=False, sample=None, pullt3=True,
bins=None):
"""
Initiliase an instance of a MultipleCaseAdder to start managing
a database update. This will get the list of cases available to
Expand Down Expand Up @@ -93,13 +95,16 @@ def __init__(self, sample_type, head=None, test_data=False, skip_demographics=Fa
self.cases_to_skip = set(self.list_of_cases) - \
set(self.cases_to_add) - \
set(self.cases_to_update)
self.update_database()
elif sample:
interpretation_list_poll = InterpretationList(sample_type=sample_type, sample=sample)
self.cases_to_poll = interpretation_list_poll.cases_to_poll
self.list_of_cases = self.fetch_api_data()
self.cases_to_update = self.list_of_cases
print(self.cases_to_update)
self.cases_to_add = []
self.cases_to_skip = []
self.update_database()
else:
# set list_of_cases to cases of interest from API
print("Fetching live API data.")
Expand All @@ -115,26 +120,56 @@ def __init__(self, sample_type, head=None, test_data=False, skip_demographics=Fa
self.total_cases_to_poll = self.total_cases_to_poll[:head]
self.num_cases_to_poll = len(self.total_cases_to_poll)

# work out how many hundreds in fetched cases
num_full_bins = self.num_cases_to_poll // 100
num_bins = num_full_bins + 1
final_bin_size = self.num_cases_to_poll - (num_full_bins * 100)

bins = []
for i in range(num_full_bins):
bins.append(
[
(100*i),
(100*i) + 100
]
) # [0, 100], [101, 200] etc
bins.append([num_full_bins * 100, None])

bin_count = 1
for b in bins:
print("Fetching cases", b[0], "to", b[1], "(bin", bin_count, "of", str(len(bins)) + ")")
self.cases_to_poll = self.total_cases_to_poll[b[0]: b[1]]
if bins:
bin_size = int(bins)
print("bin size", bin_size)
# work out how many hundreds in fetched cases
num_full_bins = self.num_cases_to_poll // bin_size

bin_ranges = []
for i in range(num_full_bins):
bin_ranges.append(
[
(bin_size * i), # e.g 0, 100
(bin_size * i) + bin_size # e.g 100, 200 - SLICE so not inclusive
]
) # [0, 100], [100, 200] etc
bin_ranges.append([num_full_bins * bin_size, None])
print("bin ranges", bin_ranges)

bin_count = 1
for b in bin_ranges:
print("Fetching cases", b[0], "to", b[1], "(bin", bin_count, "of", str(len(bin_ranges)) + ")")
self.cases_to_poll = self.total_cases_to_poll[b[0]: b[1]]

print("Fetching API JSON data for cases to poll...")
self.list_of_cases = self.fetch_api_data()
if head:
# take a certain number of cases off the top
self.list_of_cases = self.list_of_cases[:head]
print("Fetched all required CIP API data.")
print("Checking which cases to add.")
self.cases_to_add = self.check_cases_to_add()
print("Checking which cases require updating.")
self.cases_to_update = self.check_cases_to_update()
self.cases_to_skip = set(self.list_of_cases) - \
set(self.cases_to_add) - \
set(self.cases_to_update)
self.update_database()

print("Finished processing bin", bin_count, "of", len(bins))

for case in self.cases_to_add:
del case
for case in self.cases_to_update:
del case

bin_count += 1

else:
# no bins, update as normal
print("Fetching all cases.")
self.cases_to_poll = self.total_cases_to_poll
print("Fetching API JSON data for cases to poll...")
self.list_of_cases = self.fetch_api_data()
if head:
Expand All @@ -144,31 +179,24 @@ def __init__(self, sample_type, head=None, test_data=False, skip_demographics=Fa
print("Checking which cases to add.")
self.cases_to_add = self.check_cases_to_add()
print("Checking which cases require updating.")
self.cases_to_update = self.check_cases_to_update()#
self.cases_to_update = self.check_cases_to_update()
self.cases_to_skip = set(self.list_of_cases) - \
set(self.cases_to_add) - \
set(self.cases_to_update)
self.update_database()

print("Finished processing bin", bin_count, "of", len(bins))

for case in self.cases_to_add:
del case
for case in self.cases_to_update:
del case

bin_count += 1

def update_database(self):
# begin update process
# --------------------
error = None
added_cases = []
updated_cases = []
try:
logger.info("Adding cases from cases_to_add.")
print("Adding", len(self.cases_to_add), "cases")
self.add_cases()
added_cases = self.add_cases()
print("Updating", len(self.cases_to_update), "cases")
self.add_cases(update=True)
updated_cases = self.add_cases(update=True)
success = True
except Exception as e:
print("Encountered error:", e)
Expand All @@ -178,13 +206,16 @@ def update_database(self):
finally:
print("Recording update")
# record the update in ListUpdate
ListUpdate.objects.create(
listupdate = ListUpdate.objects.create(
update_time=timezone.now(),
success=success,
cases_added=len(self.cases_to_add),
cases_updated=len(self.cases_to_update),
sample_type=self.sample_type,
error=error
)
listupdate.reports_added.add(*added_cases)
listupdate.reports_updated.add(*updated_cases)

def fetch_test_data(self):
"""
Expand Down Expand Up @@ -440,11 +471,13 @@ def add_cases(self, update=False):

# finally, save jsons to disk storage
cip_api_storage = self.config['cip_api_storage']
case_reports = []
for case in cases:
ir_family = case.attribute_managers[InterpretationReportFamily].case_model.entry
latest_case = GELInterpretationReport.objects.filter(
ir_family=ir_family
).latest('polled_at_datetime')
case_reports.append(latest_case)

with open(
os.path.join(
Expand All @@ -453,6 +486,7 @@ def add_cases(self, update=False):
case.request_id + "-" + str(latest_case.archived_version)
)), 'w') as f:
json.dump(case.raw_json, f)
return case_reports


def save_new(self, model_type, model_list):
Expand Down
7 changes: 6 additions & 1 deletion gelweb/gel2mdt/management/commands/run_batch_update.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,10 @@ def add_arguments(self, parser):
parser.add_argument('--pullt3', action='store_true',
help='Include the Tier 3 variants (this is time'
' consuming!)')
parser.add_argument('--bins', action='store',
help='Whether to bin cases, and size of bins to'
'use. This increases update time but reduces RAM'
'usage for less powerful servers.')

def handle(self, *args, **options):
"""Run the MultipleCaseAdder with the supplied options."""
Expand All @@ -53,4 +57,5 @@ def handle(self, *args, **options):
head=options['case_count'],
test_data=options['test_data'],
skip_demographics=options['skip_demographics'],
pullt3=options['pullt3'])
pullt3=options['pullt3'],
bins=options['bins'])
4 changes: 3 additions & 1 deletion gelweb/gel2mdt/models.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,9 @@ class ListUpdate(models.Model):

cases_added = models.IntegerField()
cases_updated = models.IntegerField()

reports_added = models.ManyToManyField('GELInterpretationReport', related_name='reports_added')
reports_updated = models.ManyToManyField('GELInterpretationReport', related_name='reports_updated')
sample_type = models.CharField(max_length=25, blank=True, null=True)
error = models.TextField(null=True)

class Meta:
Expand Down
34 changes: 21 additions & 13 deletions gelweb/gel2mdt/tasks.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,9 +51,8 @@ def get_gel_content(user_email, ir, ir_version):
# otherwise get uname and password from a file

interpretation_reponse = PollAPI(
"cip_api_for_report", f'interpretationRequests/{ir}/{ir_version}/')
"cip_api", f'interpretation-request/{ir}/{ir_version}')
interp_json = interpretation_reponse.get_json_response()

analysis_versions = []
latest = None
if 'interpreted_genome' in interp_json:
Expand All @@ -64,16 +63,28 @@ def get_gel_content(user_email, ir, ir_version):
except ValueError as e:
latest = 1

html_report = PollAPI(
"cip_api_for_report", f"ClinicalReport/{ir}/{ir_version}/{latest}/"
)
gel_content = html_report.get_json_response(content=True)
try:
gel_json_content = json.loads(gel_content)
if gel_json_content['detail'].startswith('Not found'):
return None
if latest == 1:
print('latest', 1)
html_report = PollAPI(
"cip_api_for_report", f"ClinicalReport/{ir}/{ir_version}/{latest}"
)
gel_content = html_report.get_json_response(content=True)
else:
while latest > 0:
print('latest', latest)
html_report = PollAPI(
"cip_api_for_report", f"ClinicalReport/{ir}/{ir_version}/{latest}"
)
gel_content = html_report.get_json_response(content=True)
gel_json_content = json.loads(gel_content)
if gel_json_content['detail'].startswith('Not found') or gel_json_content['detail'].startswith(
'Method \"GET\" not allowed'):
latest -= 1
else:
break
except JSONDecodeError as e:
pass
print('JSONDecodeError')

analysis_panels = {}

Expand Down Expand Up @@ -152,7 +163,6 @@ def get_gel_content(user_email, ir, ir_version):

div_tag.insert(1, h3_tag)
div_tag.insert(2, table_tag)

gel_content = gel_content.prettify('utf-8')
with open(f"{ir}_{ir_version}_clinical_report.html", "wb") as file:
file.write(gel_content)
Expand All @@ -163,7 +173,6 @@ def get_gel_content(user_email, ir, ir_version):
msg.send()
os.remove(f"{ir}_{ir_version}_clinical_report.html")


def panel_app(gene_panel, gp_version):
'''
Returns the list of genes associated with a panel in a dictionary which are then placed in the GEL clinical report
Expand Down Expand Up @@ -576,7 +585,6 @@ def create_bokeh_barplot(names, values, title):
plot.legend.location = "top_center"
return plot


class ReportHistoryFormatter:
def __init__(self, report):
self.report = report
Expand Down
Loading

0 comments on commit 939ac04

Please sign in to comment.