+
+ Rare Disease Alert System
+
+
+
+
+
+ {name}
+Within the last week, {num} new entries for your subscribed rare diseases have been added to the {db_title} database
+| Name | +GARD ID | +Nodes Modified | +
|---|---|---|
| {name} | +{gardId} | +{num} | +
Results gathered within the time period of {date_start}-{date_end}
++
+ 
+
+
+ """.format(date_start=data['update_date_start'],date_end=data['update_date_end'])
+
+ print(html)
+ return (full_msg,html)
+
+def send_mail(type, data):
+ print(f"[{data['total']}, {data['email']}]")
+ if data['total'] > 0 and data['email'] == 'timothy.sheils@ncats.nih.gov' or data['email'] == 'zhuqianzq@gmail.com': #TEST
+
+ data['email'] = 'devon.leadman@nih.gov' # TEST EMAIL
+
+ if type == "clinical":
+ txt,html = fill_template(type,data)
+ alert.send_email('RDAS-Alert: Clinical Trial update regarding your subscriptions', txt, data['email'], html=html) #data['email'] in place of email
+ print('[Email Sent...]')
+
+ if type == "pubmed":
+ txt,html = fill_template(type,data)
+ alert.send_email('RDAS-Alert: Publication update regarding your subscriptions', txt, data['email'], html=html)
+ print('[Email Sent...]')
+
+ if type == "grant":
+ txt,html = fill_template(type,data)
+ alert.send_email('RDAS-Alert: Funded Project update regarding your subscriptions', txt, data['email'], html=html)
+ print('[Email Sent...]')
+
+def get_stats(type, gards, date_start=datetime.today().strftime('%m/%d/%y'), date_end=datetime.today().strftime('%m/%d/%y')):
+ db = AlertCypher(type)
+ return_data = dict()
+
+ date_start_string = date_start
+ date_end_string = date_end
+ date_start_obj = datetime.strptime(date_start, '%m/%d/%y')
+ date_end_obj = datetime.strptime(date_end, '%m/%d/%y')
+
+ date_list = pd.date_range(date_start_obj, date_end_obj, freq='D').strftime('%m/%d/%y').to_list()
+
+ print(f'Searching for nodes created between {date_start_string} and {date_end_string}')
+
+ convert = {'clinical':['ClinicalTrial','GARD','GardId'], 'pubmed':['Article','GARD','GardId'], 'grant':['Project','GARD','GardId']}
+ connect_to_gard = {'clinical':'--(:Condition)--(:Annotation)--','pubmed':'--','grant':'--'}
+
+ query = 'MATCH (x:{node}){connection}(y:{gardnode}) WHERE x.DateCreatedRDAS IN {date_list} AND y.{property} IN {list} RETURN COUNT(x)'.format(node=convert[type][0], gardnode=convert[type][1], property=convert[type][2], list=list(gards.keys()), date_list=date_list, connection=connect_to_gard[type])
+
+ response = db.run(query)
+ return_data['total'] = response.data()[0]['COUNT(x)']
+
+ for gard in gards.keys():
+ response = db.run('MATCH (x:{node}){connection}(y:{gardnode}) WHERE x.DateCreatedRDAS IN {date_list} AND y.{property} = \"{gard}\" RETURN COUNT(x)'.format(node=convert[type][0], gardnode=convert[type][1], property=convert[type][2], gard=gard, date_list=date_list, connection=connect_to_gard[type]))
+ return_data[gard] = {'name':gards[gard],'num':response.data()[0]['COUNT(x)']}
+
+ return_data['update_date_end'] = date_end_string
+ return_data['update_date_start'] = date_start_string
+
+ return return_data
+
+def trigger_email(type,date_start=None,date_end=None):
+ convert = {'clinical':'trials', 'pubmed':'articles', 'grant':'grants'}
+ user_data = dict()
+ cred = credentials.Certificate(sysvars.firebase_key_path)
+ firebase_admin.initialize_app(cred)
+ firestore_db = firestore.client()
+
+ firestore_docs = firestore_db.collection(u'users').stream()
+
+ for doc in firestore_docs:
+ if doc.exists:
+ user_data[doc.id] = doc.to_dict()
+ else:
+ print('Document Doesnt Exist')
+
+ for firestore_user, data in user_data.items():
+ subscript_gard = dict()
+ for subscript in data['subscriptions']:
+ try:
+ if convert[type] in subscript['alerts']:
+ subscript_gard[subscript['gardID']] = subscript['diseaseName']
+ except KeyError:
+ print('')
+ pass
+ users = auth.list_users()
+ if users:
+ users = users.iterate_all()
+ for user in users:
+ uid = user.uid
+ if uid == firestore_user and len(subscript_gard) > 0:
+ if not date_start and not date_end:
+ update_data = get_stats(type, subscript_gard)
+ elif date_start and date_end:
+ update_data = get_stats(type, subscript_gard, date_start=date_start, date_end=date_end)
+ elif date_start:
+ update_data = get_stats(type, subscript_gard, date_start=date_start)
+ elif date_end:
+ update_data = get_stats(type, subscript_gard, date_end=date_end)
+
+ update_data['email'] = user.email
+ update_data['name'] = user_data[uid]['displayName']
+ update_data['subscriptions'] = list(subscript_gard.keys())
+ send_mail(type, update_data)
+
+#trigger_email(sysvars.pm_db, date_start='12/07/22') #TEST
diff --git a/emails/alert.py b/emails/alert.py
old mode 100644
new mode 100755
diff --git a/emails/email_template1.html b/emails/email_template1.html
new file mode 100755
index 0000000..3bbb6ac
--- /dev/null
+++ b/emails/email_template1.html
@@ -0,0 +1,32 @@
+
+
+
+
+
+
+
+
+
+
+
diff --git a/emails/email_test.py b/emails/email_test.py
new file mode 100755
index 0000000..bc1ae73
--- /dev/null
+++ b/emails/email_test.py
@@ -0,0 +1,50 @@
+
+import sys
+
+sys.path.append('/home/aom2/RDAS')
+sys.path.append('/home/aom2/RDAS/emails')
+import sysvars
+import smtplib
+from email.mime.text import MIMEText
+from email.mime.multipart import MIMEMultipart
+from jinja2 import Environment, FileSystemLoader
+import os
+
+
+def send_email(subject, html, recipient):
+ print("sending emails to::", recipient)
+ sender = "" # Replace with your email
+ password = "" # Replace with your email password
+
+ # Set up the email
+ msg = MIMEMultipart('alternative')
+ msg['From'] = ""
+ # msg['To'] = ""
+ msg['Subject'] = subject
+
+ # Attach both plain text and HTML parts
+ # part1 = MIMEText(text, 'plain')
+ part2 = MIMEText(html, 'html')
+ # msg.attach(part1)
+ msg.attach(part2)
+
+ # Send the email
+ server = smtplib.SMTP('', 587) # Replace with SMTP server and port
+ server.starttls()
+ server.login(sender, password)
+ text = msg.as_string()
+ server.sendmail(sender, recipient, msg.as_string())
+ server.quit()
+
+
+# def render_template(filename, data={}):
+# # template_dir = "path/to/templates"
+# template_dir = os.getcwd()
+# env = Environment(loader=FileSystemLoader(template_dir))
+# template_path = filename # Relative to the template_dir
+# template = env.get_template(template_path)
+# return template.render(data=data)
+
+# the fill_template method was remove, becaus the generation of message was integrated into the html template.
+
+
diff --git a/emails/ses_firebase.py b/emails/ses_firebase.py
old mode 100644
new mode 100755
diff --git a/emails/test_ses_firebase.py b/emails/test_ses_firebase.py
new file mode 100755
index 0000000..ecfd572
--- /dev/null
+++ b/emails/test_ses_firebase.py
@@ -0,0 +1,190 @@
+import glob
+import os
+import sys
+import json
+workspace = os.path.dirname(os.path.abspath(__file__))
+sys.path.append(workspace)
+# sys.path.append(os.getcwd())
+sys.path.append('/home/aom2/RDAS')
+sys.path.append('/home/aom2/RDAS/emails')
+import sysvars
+from AlertCypher import AlertCypher
+from datetime import date,datetime
+from jinja2 import Environment, FileSystemLoader
+import pandas as pd
+from dateutil.relativedelta import relativedelta
+import firebase_admin
+from firebase_admin import auth
+from firebase_admin import credentials
+from firebase_admin import firestore
+import alert
+import email_test
+
+
+prefix = sysvars.db_prefix # you can set the db_prefix in sysvars.py
+
+
+def render_template(filename, data={}):
+ env = Environment(loader=FileSystemLoader(f'{sysvars.base_path}emails/'))
+ template = env.get_template(filename)
+ rendered_content = template.render(data=data)
+ return rendered_content
+
+def send_mail(type, data):
+ # Define the tabs dictionary and txt_db
+ tabs = {prefix +'clinical': 'trials', prefix +'grant': 'project', prefix +'pubmed': 'nonepi-articles'}
+ txt_db = {prefix +'clinical': 'clinical trial', prefix +'grant': 'funded project', prefix +'pubmed': 'publication'}
+
+ # Add tabs and type to the data dictionary
+ data['tabs'] = tabs
+ data["db_title"]=str(txt_db[type])
+
+ if data['total'] > 0 and data['email'] == 'timothy.sheils@ncats.nih.gov' or data['email'] == 'zhuqianzq@gmail.com':# for testing
+
+ data['email'] = 'minghui.ao@nih.gov' # TEST EMAIL
+ html_content = render_template('email_template1.html', data=data)
+ email_test.send_email(f'RDAS-Alert: {str(txt_db[type])} update regarding your subscriptions', html_content, data['email'])# change to your alert.py sending email method.you may need to adjust your method abit to read in these parameters.
+ print("finish sending enail")
+
+def get_stats(type, gards, date_start=datetime.today().strftime('%m/%d/%y'), date_end=datetime.today().strftime('%m/%d/%y')):
+ db = AlertCypher(type)
+ return_data = dict()
+ date_start_string = date_start
+ date_end_string = date_end
+ date_start_obj = datetime.strptime(date_start, '%m/%d/%y')
+ date_end_obj = datetime.strptime(date_end, '%m/%d/%y')
+
+ date_list = pd.date_range(date_start_obj, date_end_obj, freq='D').strftime('%m/%d/%y').to_list()
+
+ convert = {prefix+'clinical':['ClinicalTrial','GARD','GardId'], prefix+'pubmed':['Article','GARD','GardId'], prefix+'grant':['Project','GARD','GardId']}
+ connect_to_gard = {prefix+'clinical':'--(:Condition)--(:Annotation)--',prefix+'pubmed':'--',prefix+'grant':'--'}
+
+ query = 'MATCH (x:{node}){connection}(y:{gardnode}) WHERE x.DateCreatedRDAS IN {date_list} AND y.{property} IN {list} RETURN COUNT(x)'.format(node=convert[type][0], gardnode=convert[type][1], property=convert[type][2], list=list(gards.keys()), date_list=date_list, connection=connect_to_gard[type])
+
+ response = db.run(query)
+ result = response.single()
+ return_data['total'] = result['COUNT(x)']
+
+ for gard in gards.keys():
+ query_1='MATCH (x:{node}){connection}(y:{gardnode}) WHERE x.DateCreatedRDAS IN {date_list} AND y.{property} = \"{gard}\" RETURN COUNT(x)'.format(node=convert[type][0], gardnode=convert[type][1], property=convert[type][2], gard=gard, date_list=date_list, connection=connect_to_gard[type])
+ response = db.run(query_1)
+ result = response.single()
+ return_data[gard] = {'name':gards[gard],'num':result['COUNT(x)']}
+
+ return_data['update_date_end'] = date_end_string
+ return_data['update_date_start'] = date_start_string
+
+ return return_data
+
+
+# def trigger_email(type,date_start=None,date_end=None):
+# convert = {prefix+'clinical':'trials', prefix+'pubmed':'articles', prefix+'grant':'grants'}
+# user_data = dict()
+# cred = credentials.Certificate(sysvars.firebase_key_path)
+# firebase_admin.initialize_app(cred)
+# firestore_db = firestore.client()
+# firestore_docs = firestore_db.collection(u'users').stream()
+
+# # get user subscription data
+# for doc in firestore_docs:
+# if doc.exists:
+# user_data[doc.id] = doc.to_dict()
+# else:
+# print('Document Doesnt Exist')
+
+# for firestore_user, data in user_data.items():
+# subscript_gard = dict()
+# for subscript in data['subscriptions']:
+
+# if convert[type] in subscript['alerts']:
+# print("user_data::", data)
+# print("subscript::",subscript,"\n")
+
+# if 'diseaseName' not in subscript:
+
+# subscript_gard[subscript['gardID']] = subscript['gardID']
+# else:
+# subscript_gard[subscript['gardID']] = subscript['diseaseName']
+
+# # get user emails
+# users = auth.list_users()
+
+# if users:
+# users = users.iterate_all()
+# for user in users:
+# uid = user.uid
+
+# if uid == firestore_user and len(subscript_gard) > 0:
+
+# if not date_start and not date_end:
+# update_data = get_stats(type, subscript_gard)
+# elif date_start and date_end:
+# update_data = get_stats(type, subscript_gard, date_start=date_start, date_end=date_end)
+# elif date_start:
+# update_data = get_stats(type, subscript_gard, date_start=date_start)
+# elif date_end:
+# update_data = get_stats(type, subscript_gard, date_end=date_end)
+
+# update_data['email'] = user.email
+# update_data['name'] = user_data[uid].get('displayName',"")
+# update_data['subscriptions'] = list(subscript_gard.keys())
+# print("update_data::",update_data)
+# if update_data["total"]>0: # only send email to user if there is any updates
+# send_mail(type, update_data)
+
+
+
+# the trigger_email function was rewrite to avoid the three nested for loops.
+def trigger_email(firestore_db,type,date_start=None,date_end=None):
+ convert = {prefix+'clinical':'trials', prefix+'pubmed':'articles', prefix+'grant':'grants'}
+ user_data = dict()
+ firestore_docs = firestore_db.collection(u'users').stream()
+
+ # get user subscription data here to avoid 3 nested for loops
+ for doc in firestore_docs:
+ if doc.exists:
+ user_data[doc.id] = doc.to_dict()
+ else:
+ print('Document Doesnt Exist')
+
+ users = auth.list_users()
+ user_info={}
+ if users:
+ users = users.iterate_all()
+ for user in users:
+ uid = user.uid
+ user_info[user.uid]=user
+
+ for firestore_user, data in user_data.items():
+ subscript_gard = dict()
+ for subscript in data['subscriptions']:
+ if convert[type] in subscript['alerts']:
+ if 'diseaseName' not in subscript:
+ subscript_gard[subscript['gardID']] = ""
+ else:
+ subscript_gard[subscript['gardID']] = subscript['diseaseName']
+
+ # get user emails
+ user=user_info.get(firestore_user,None)
+ if user:
+ uid=user.uid
+ # print("uid == firestore_user::",uid == firestore_user,len(subscript_gard))
+ if uid == firestore_user and len(subscript_gard) > 0:
+
+ if not date_start and not date_end:
+ update_data = get_stats(type, subscript_gard)
+ elif date_start and date_end:
+ update_data = get_stats(type, subscript_gard, date_start=date_start, date_end=date_end)
+ elif date_start:
+ update_data = get_stats(type, subscript_gard, date_start=date_start)
+ elif date_end:
+ update_data = get_stats(type, subscript_gard, date_end=date_end)
+
+ update_data['email'] = user.email
+ update_data['name'] = user_data[uid].get('displayName',"")
+ update_data['subscriptions'] = list(subscript_gard.keys())
+ # print("update_data::",update_data)
+ if update_data["total"]>0: # only send email to user if there is any updates
+ send_mail(type, update_data)
+
+# trigger_email(sysvars.ct_db, date_start='12/07/22') #TEST. you can put this to the start_dev. so when there are any db upates, it will trigger emails
diff --git a/etc/acronym_algorithm_detection.csv b/etc/acronym_algorithm_detection.csv
old mode 100644
new mode 100755
diff --git a/etc/add_new_epi_model.py b/etc/add_new_epi_model.py
old mode 100644
new mode 100755
diff --git a/etc/compare_db.py b/etc/compare_db.py
old mode 100644
new mode 100755
diff --git a/etc/fda.out b/etc/fda.out
old mode 100644
new mode 100755
diff --git a/etc/fda_gard_mapping_6_14_2023.csv b/etc/fda_gard_mapping_6_14_2023.csv
old mode 100644
new mode 100755
diff --git a/etc/fixpubtatortext.py b/etc/fixpubtatortext.py
old mode 100644
new mode 100755
diff --git a/etc/gardfix.py b/etc/gardfix.py
old mode 100644
new mode 100755
diff --git a/etc/map_fda.py b/etc/map_fda.py
old mode 100644
new mode 100755
diff --git a/etc/map_fda_new.py b/etc/map_fda_new.py
old mode 100644
new mode 100755
diff --git a/etc/metamap_cond.py b/etc/metamap_cond.py
old mode 100644
new mode 100755
diff --git a/etc/metamap_fda_out.json b/etc/metamap_fda_out.json
old mode 100644
new mode 100755
diff --git a/etc/orphan_substance_public_records_6_14_2023.csv b/etc/orphan_substance_public_records_6_14_2023.csv
old mode 100644
new mode 100755
diff --git a/etc/orphan_substance_public_records_6_14_2023.xlsx b/etc/orphan_substance_public_records_6_14_2023.xlsx
old mode 100644
new mode 100755
diff --git a/etc/remove_article_fps.py b/etc/remove_article_fps.py
old mode 100644
new mode 100755
diff --git a/etc/remove_old_trials.py b/etc/remove_old_trials.py
old mode 100644
new mode 100755
diff --git a/etc/runmap.py b/etc/runmap.py
old mode 100644
new mode 100755
diff --git a/etc/rxnorm_mapper.py b/etc/rxnorm_mapper.py
old mode 100644
new mode 100755
diff --git a/etc/sim_match.py b/etc/sim_match.py
old mode 100644
new mode 100755
diff --git a/file_transfer.py b/file_transfer.py
old mode 100644
new mode 100755
diff --git a/generate_dump.py b/generate_dump.py
old mode 100644
new mode 100755
diff --git a/grant_2024/init.py b/grant_2024/init.py
old mode 100644
new mode 100755
diff --git a/grant_2024/methods.py b/grant_2024/methods.py
old mode 100644
new mode 100755
diff --git a/grant_2024/update.py b/grant_2024/update.py
old mode 100644
new mode 100755
diff --git a/img/RDAS_logo_color.png b/img/RDAS_logo_color.png
old mode 100644
new mode 100755
diff --git a/img/RDAS_logo_color.svg b/img/RDAS_logo_color.svg
old mode 100644
new mode 100755
diff --git a/requirements.txt b/requirements.txt
old mode 100644
new mode 100755
diff --git a/start_prod.py b/start_prod.py
old mode 100644
new mode 100755
diff --git a/sysvars.py b/sysvars.py
index 0fefab6..43cc129 100644
--- a/sysvars.py
+++ b/sysvars.py
@@ -41,10 +41,10 @@
db_abbrevs2 = {ct_db:'ct', pm_db:'pm', gnt_db:'gnt'}
# Paths to database creation and update source files
-ct_files_path = '{base_path}RDAS.CTKG/src/'.format(base_path=base_path)
-pm_files_path = '{base_path}RDAS.PAKG/src/'.format(base_path=base_path)
-gnt_files_path = '{base_path}RDAS.GFKG/src/'.format(base_path=base_path)
-gard_files_path = '{base_path}RDAS.GARD/src/'.format(base_path=base_path)
+ct_files_path = '{base_path}RDAS_CTKG/src/'.format(base_path=base_path)
+pm_files_path = '{base_path}RDAS_PAKG/src/'.format(base_path=base_path)
+gnt_files_path = '{base_path}RDAS_GFKG/src/'.format(base_path=base_path)
+gard_files_path = '{base_path}RDAS_GARD/src/'.format(base_path=base_path)
# Database names being used on the current server
convert = {ct_db:'trials', pm_db:'articles', gnt_db:'grants'}
diff --git a/transfer/README.md b/transfer/README.md
old mode 100644
new mode 100755
+
+
+ """.format(date_start=data['update_date_start'],date_end=data['update_date_end'])
+
+ print(html)
+ return (full_msg,html)
+
+def send_mail(type, data):
+ print(f"[{data['total']}, {data['email']}]")
+ if data['total'] > 0 and data['email'] == 'timothy.sheils@ncats.nih.gov' or data['email'] == 'zhuqianzq@gmail.com': #TEST
+
+ data['email'] = 'devon.leadman@nih.gov' # TEST EMAIL
+
+ if type == "clinical":
+ txt,html = fill_template(type,data)
+ alert.send_email('RDAS-Alert: Clinical Trial update regarding your subscriptions', txt, data['email'], html=html) #data['email'] in place of email
+ print('[Email Sent...]')
+
+ if type == "pubmed":
+ txt,html = fill_template(type,data)
+ alert.send_email('RDAS-Alert: Publication update regarding your subscriptions', txt, data['email'], html=html)
+ print('[Email Sent...]')
+
+ if type == "grant":
+ txt,html = fill_template(type,data)
+ alert.send_email('RDAS-Alert: Funded Project update regarding your subscriptions', txt, data['email'], html=html)
+ print('[Email Sent...]')
+
+def get_stats(type, gards, date_start=datetime.today().strftime('%m/%d/%y'), date_end=datetime.today().strftime('%m/%d/%y')):
+ db = AlertCypher(type)
+ return_data = dict()
+
+ date_start_string = date_start
+ date_end_string = date_end
+ date_start_obj = datetime.strptime(date_start, '%m/%d/%y')
+ date_end_obj = datetime.strptime(date_end, '%m/%d/%y')
+
+ date_list = pd.date_range(date_start_obj, date_end_obj, freq='D').strftime('%m/%d/%y').to_list()
+
+ print(f'Searching for nodes created between {date_start_string} and {date_end_string}')
+
+ convert = {'clinical':['ClinicalTrial','GARD','GardId'], 'pubmed':['Article','GARD','GardId'], 'grant':['Project','GARD','GardId']}
+ connect_to_gard = {'clinical':'--(:Condition)--(:Annotation)--','pubmed':'--','grant':'--'}
+
+ query = 'MATCH (x:{node}){connection}(y:{gardnode}) WHERE x.DateCreatedRDAS IN {date_list} AND y.{property} IN {list} RETURN COUNT(x)'.format(node=convert[type][0], gardnode=convert[type][1], property=convert[type][2], list=list(gards.keys()), date_list=date_list, connection=connect_to_gard[type])
+
+ response = db.run(query)
+ return_data['total'] = response.data()[0]['COUNT(x)']
+
+ for gard in gards.keys():
+ response = db.run('MATCH (x:{node}){connection}(y:{gardnode}) WHERE x.DateCreatedRDAS IN {date_list} AND y.{property} = \"{gard}\" RETURN COUNT(x)'.format(node=convert[type][0], gardnode=convert[type][1], property=convert[type][2], gard=gard, date_list=date_list, connection=connect_to_gard[type]))
+ return_data[gard] = {'name':gards[gard],'num':response.data()[0]['COUNT(x)']}
+
+ return_data['update_date_end'] = date_end_string
+ return_data['update_date_start'] = date_start_string
+
+ return return_data
+
+def trigger_email(type,date_start=None,date_end=None):
+ convert = {'clinical':'trials', 'pubmed':'articles', 'grant':'grants'}
+ user_data = dict()
+ cred = credentials.Certificate(sysvars.firebase_key_path)
+ firebase_admin.initialize_app(cred)
+ firestore_db = firestore.client()
+
+ firestore_docs = firestore_db.collection(u'users').stream()
+
+ for doc in firestore_docs:
+ if doc.exists:
+ user_data[doc.id] = doc.to_dict()
+ else:
+ print('Document Doesnt Exist')
+
+ for firestore_user, data in user_data.items():
+ subscript_gard = dict()
+ for subscript in data['subscriptions']:
+ try:
+ if convert[type] in subscript['alerts']:
+ subscript_gard[subscript['gardID']] = subscript['diseaseName']
+ except KeyError:
+ print('')
+ pass
+ users = auth.list_users()
+ if users:
+ users = users.iterate_all()
+ for user in users:
+ uid = user.uid
+ if uid == firestore_user and len(subscript_gard) > 0:
+ if not date_start and not date_end:
+ update_data = get_stats(type, subscript_gard)
+ elif date_start and date_end:
+ update_data = get_stats(type, subscript_gard, date_start=date_start, date_end=date_end)
+ elif date_start:
+ update_data = get_stats(type, subscript_gard, date_start=date_start)
+ elif date_end:
+ update_data = get_stats(type, subscript_gard, date_end=date_end)
+
+ update_data['email'] = user.email
+ update_data['name'] = user_data[uid]['displayName']
+ update_data['subscriptions'] = list(subscript_gard.keys())
+ send_mail(type, update_data)
+
+#trigger_email(sysvars.pm_db, date_start='12/07/22') #TEST
diff --git a/emails/alert.py b/emails/alert.py
old mode 100644
new mode 100755
diff --git a/emails/email_template1.html b/emails/email_template1.html
new file mode 100755
index 0000000..3bbb6ac
--- /dev/null
+++ b/emails/email_template1.html
@@ -0,0 +1,32 @@
+
+
+
+
+
+
+ Rare Disease Alert System
+{{ name }}
+You have {{ data['total'] }} new entries for your subscribed rare diseases in the {{ data['db_title'] }} database
+| Name | +GARD ID | +Nodes Modified | +
|---|---|---|
| {{ data[gard_id]['name'] }} | +{{ gard_id }} | +{{ data[gard_id]['num'] }} | +
Results gathered within the time period of {{ data['update_date_start'] }} - {{ data['update_date_end'] }}
+
+
+
diff --git a/emails/email_test.py b/emails/email_test.py
new file mode 100755
index 0000000..bc1ae73
--- /dev/null
+++ b/emails/email_test.py
@@ -0,0 +1,50 @@
+
+import sys
+
+sys.path.append('/home/aom2/RDAS')
+sys.path.append('/home/aom2/RDAS/emails')
+import sysvars
+import smtplib
+from email.mime.text import MIMEText
+from email.mime.multipart import MIMEMultipart
+from jinja2 import Environment, FileSystemLoader
+import os
+
+
+def send_email(subject, html, recipient):
+ print("sending emails to::", recipient)
+ sender = "" # Replace with your email
+ password = "" # Replace with your email password
+
+ # Set up the email
+ msg = MIMEMultipart('alternative')
+ msg['From'] = ""
+ # msg['To'] = ""
+ msg['Subject'] = subject
+
+ # Attach both plain text and HTML parts
+ # part1 = MIMEText(text, 'plain')
+ part2 = MIMEText(html, 'html')
+ # msg.attach(part1)
+ msg.attach(part2)
+
+ # Send the email
+ server = smtplib.SMTP('', 587) # Replace with SMTP server and port
+ server.starttls()
+ server.login(sender, password)
+ text = msg.as_string()
+ server.sendmail(sender, recipient, msg.as_string())
+ server.quit()
+
+
+# def render_template(filename, data={}):
+# # template_dir = "path/to/templates"
+# template_dir = os.getcwd()
+# env = Environment(loader=FileSystemLoader(template_dir))
+# template_path = filename # Relative to the template_dir
+# template = env.get_template(template_path)
+# return template.render(data=data)
+
+# the fill_template method was remove, becaus the generation of message was integrated into the html template.
+
+
diff --git a/emails/ses_firebase.py b/emails/ses_firebase.py
old mode 100644
new mode 100755
diff --git a/emails/test_ses_firebase.py b/emails/test_ses_firebase.py
new file mode 100755
index 0000000..ecfd572
--- /dev/null
+++ b/emails/test_ses_firebase.py
@@ -0,0 +1,190 @@
+import glob
+import os
+import sys
+import json
+workspace = os.path.dirname(os.path.abspath(__file__))
+sys.path.append(workspace)
+# sys.path.append(os.getcwd())
+sys.path.append('/home/aom2/RDAS')
+sys.path.append('/home/aom2/RDAS/emails')
+import sysvars
+from AlertCypher import AlertCypher
+from datetime import date,datetime
+from jinja2 import Environment, FileSystemLoader
+import pandas as pd
+from dateutil.relativedelta import relativedelta
+import firebase_admin
+from firebase_admin import auth
+from firebase_admin import credentials
+from firebase_admin import firestore
+import alert
+import email_test
+
+
+prefix = sysvars.db_prefix # you can set the db_prefix in sysvars.py
+
+
+def render_template(filename, data={}):
+ env = Environment(loader=FileSystemLoader(f'{sysvars.base_path}emails/'))
+ template = env.get_template(filename)
+ rendered_content = template.render(data=data)
+ return rendered_content
+
+def send_mail(type, data):
+ # Define the tabs dictionary and txt_db
+ tabs = {prefix +'clinical': 'trials', prefix +'grant': 'project', prefix +'pubmed': 'nonepi-articles'}
+ txt_db = {prefix +'clinical': 'clinical trial', prefix +'grant': 'funded project', prefix +'pubmed': 'publication'}
+
+ # Add tabs and type to the data dictionary
+ data['tabs'] = tabs
+ data["db_title"]=str(txt_db[type])
+
+ if data['total'] > 0 and data['email'] == 'timothy.sheils@ncats.nih.gov' or data['email'] == 'zhuqianzq@gmail.com':# for testing
+
+ data['email'] = 'minghui.ao@nih.gov' # TEST EMAIL
+ html_content = render_template('email_template1.html', data=data)
+ email_test.send_email(f'RDAS-Alert: {str(txt_db[type])} update regarding your subscriptions', html_content, data['email'])# change to your alert.py sending email method.you may need to adjust your method abit to read in these parameters.
+ print("finish sending enail")
+
+def get_stats(type, gards, date_start=datetime.today().strftime('%m/%d/%y'), date_end=datetime.today().strftime('%m/%d/%y')):
+ db = AlertCypher(type)
+ return_data = dict()
+ date_start_string = date_start
+ date_end_string = date_end
+ date_start_obj = datetime.strptime(date_start, '%m/%d/%y')
+ date_end_obj = datetime.strptime(date_end, '%m/%d/%y')
+
+ date_list = pd.date_range(date_start_obj, date_end_obj, freq='D').strftime('%m/%d/%y').to_list()
+
+ convert = {prefix+'clinical':['ClinicalTrial','GARD','GardId'], prefix+'pubmed':['Article','GARD','GardId'], prefix+'grant':['Project','GARD','GardId']}
+ connect_to_gard = {prefix+'clinical':'--(:Condition)--(:Annotation)--',prefix+'pubmed':'--',prefix+'grant':'--'}
+
+ query = 'MATCH (x:{node}){connection}(y:{gardnode}) WHERE x.DateCreatedRDAS IN {date_list} AND y.{property} IN {list} RETURN COUNT(x)'.format(node=convert[type][0], gardnode=convert[type][1], property=convert[type][2], list=list(gards.keys()), date_list=date_list, connection=connect_to_gard[type])
+
+ response = db.run(query)
+ result = response.single()
+ return_data['total'] = result['COUNT(x)']
+
+ for gard in gards.keys():
+ query_1='MATCH (x:{node}){connection}(y:{gardnode}) WHERE x.DateCreatedRDAS IN {date_list} AND y.{property} = \"{gard}\" RETURN COUNT(x)'.format(node=convert[type][0], gardnode=convert[type][1], property=convert[type][2], gard=gard, date_list=date_list, connection=connect_to_gard[type])
+ response = db.run(query_1)
+ result = response.single()
+ return_data[gard] = {'name':gards[gard],'num':result['COUNT(x)']}
+
+ return_data['update_date_end'] = date_end_string
+ return_data['update_date_start'] = date_start_string
+
+ return return_data
+
+
+# def trigger_email(type,date_start=None,date_end=None):
+# convert = {prefix+'clinical':'trials', prefix+'pubmed':'articles', prefix+'grant':'grants'}
+# user_data = dict()
+# cred = credentials.Certificate(sysvars.firebase_key_path)
+# firebase_admin.initialize_app(cred)
+# firestore_db = firestore.client()
+# firestore_docs = firestore_db.collection(u'users').stream()
+
+# # get user subscription data
+# for doc in firestore_docs:
+# if doc.exists:
+# user_data[doc.id] = doc.to_dict()
+# else:
+# print('Document Doesnt Exist')
+
+# for firestore_user, data in user_data.items():
+# subscript_gard = dict()
+# for subscript in data['subscriptions']:
+
+# if convert[type] in subscript['alerts']:
+# print("user_data::", data)
+# print("subscript::",subscript,"\n")
+
+# if 'diseaseName' not in subscript:
+
+# subscript_gard[subscript['gardID']] = subscript['gardID']
+# else:
+# subscript_gard[subscript['gardID']] = subscript['diseaseName']
+
+# # get user emails
+# users = auth.list_users()
+
+# if users:
+# users = users.iterate_all()
+# for user in users:
+# uid = user.uid
+
+# if uid == firestore_user and len(subscript_gard) > 0:
+
+# if not date_start and not date_end:
+# update_data = get_stats(type, subscript_gard)
+# elif date_start and date_end:
+# update_data = get_stats(type, subscript_gard, date_start=date_start, date_end=date_end)
+# elif date_start:
+# update_data = get_stats(type, subscript_gard, date_start=date_start)
+# elif date_end:
+# update_data = get_stats(type, subscript_gard, date_end=date_end)
+
+# update_data['email'] = user.email
+# update_data['name'] = user_data[uid].get('displayName',"")
+# update_data['subscriptions'] = list(subscript_gard.keys())
+# print("update_data::",update_data)
+# if update_data["total"]>0: # only send email to user if there is any updates
+# send_mail(type, update_data)
+
+
+
+# the trigger_email function was rewrite to avoid the three nested for loops.
+def trigger_email(firestore_db,type,date_start=None,date_end=None):
+ convert = {prefix+'clinical':'trials', prefix+'pubmed':'articles', prefix+'grant':'grants'}
+ user_data = dict()
+ firestore_docs = firestore_db.collection(u'users').stream()
+
+ # get user subscription data here to avoid 3 nested for loops
+ for doc in firestore_docs:
+ if doc.exists:
+ user_data[doc.id] = doc.to_dict()
+ else:
+ print('Document Doesnt Exist')
+
+ users = auth.list_users()
+ user_info={}
+ if users:
+ users = users.iterate_all()
+ for user in users:
+ uid = user.uid
+ user_info[user.uid]=user
+
+ for firestore_user, data in user_data.items():
+ subscript_gard = dict()
+ for subscript in data['subscriptions']:
+ if convert[type] in subscript['alerts']:
+ if 'diseaseName' not in subscript:
+ subscript_gard[subscript['gardID']] = ""
+ else:
+ subscript_gard[subscript['gardID']] = subscript['diseaseName']
+
+ # get user emails
+ user=user_info.get(firestore_user,None)
+ if user:
+ uid=user.uid
+ # print("uid == firestore_user::",uid == firestore_user,len(subscript_gard))
+ if uid == firestore_user and len(subscript_gard) > 0:
+
+ if not date_start and not date_end:
+ update_data = get_stats(type, subscript_gard)
+ elif date_start and date_end:
+ update_data = get_stats(type, subscript_gard, date_start=date_start, date_end=date_end)
+ elif date_start:
+ update_data = get_stats(type, subscript_gard, date_start=date_start)
+ elif date_end:
+ update_data = get_stats(type, subscript_gard, date_end=date_end)
+
+ update_data['email'] = user.email
+ update_data['name'] = user_data[uid].get('displayName',"")
+ update_data['subscriptions'] = list(subscript_gard.keys())
+ # print("update_data::",update_data)
+ if update_data["total"]>0: # only send email to user if there is any updates
+ send_mail(type, update_data)
+
+# trigger_email(sysvars.ct_db, date_start='12/07/22') #TEST. you can put this to the start_dev. so when there are any db upates, it will trigger emails
diff --git a/etc/acronym_algorithm_detection.csv b/etc/acronym_algorithm_detection.csv
old mode 100644
new mode 100755
diff --git a/etc/add_new_epi_model.py b/etc/add_new_epi_model.py
old mode 100644
new mode 100755
diff --git a/etc/compare_db.py b/etc/compare_db.py
old mode 100644
new mode 100755
diff --git a/etc/fda.out b/etc/fda.out
old mode 100644
new mode 100755
diff --git a/etc/fda_gard_mapping_6_14_2023.csv b/etc/fda_gard_mapping_6_14_2023.csv
old mode 100644
new mode 100755
diff --git a/etc/fixpubtatortext.py b/etc/fixpubtatortext.py
old mode 100644
new mode 100755
diff --git a/etc/gardfix.py b/etc/gardfix.py
old mode 100644
new mode 100755
diff --git a/etc/map_fda.py b/etc/map_fda.py
old mode 100644
new mode 100755
diff --git a/etc/map_fda_new.py b/etc/map_fda_new.py
old mode 100644
new mode 100755
diff --git a/etc/metamap_cond.py b/etc/metamap_cond.py
old mode 100644
new mode 100755
diff --git a/etc/metamap_fda_out.json b/etc/metamap_fda_out.json
old mode 100644
new mode 100755
diff --git a/etc/orphan_substance_public_records_6_14_2023.csv b/etc/orphan_substance_public_records_6_14_2023.csv
old mode 100644
new mode 100755
diff --git a/etc/orphan_substance_public_records_6_14_2023.xlsx b/etc/orphan_substance_public_records_6_14_2023.xlsx
old mode 100644
new mode 100755
diff --git a/etc/remove_article_fps.py b/etc/remove_article_fps.py
old mode 100644
new mode 100755
diff --git a/etc/remove_old_trials.py b/etc/remove_old_trials.py
old mode 100644
new mode 100755
diff --git a/etc/runmap.py b/etc/runmap.py
old mode 100644
new mode 100755
diff --git a/etc/rxnorm_mapper.py b/etc/rxnorm_mapper.py
old mode 100644
new mode 100755
diff --git a/etc/sim_match.py b/etc/sim_match.py
old mode 100644
new mode 100755
diff --git a/file_transfer.py b/file_transfer.py
old mode 100644
new mode 100755
diff --git a/generate_dump.py b/generate_dump.py
old mode 100644
new mode 100755
diff --git a/grant_2024/init.py b/grant_2024/init.py
old mode 100644
new mode 100755
diff --git a/grant_2024/methods.py b/grant_2024/methods.py
old mode 100644
new mode 100755
diff --git a/grant_2024/update.py b/grant_2024/update.py
old mode 100644
new mode 100755
diff --git a/img/RDAS_logo_color.png b/img/RDAS_logo_color.png
old mode 100644
new mode 100755
diff --git a/img/RDAS_logo_color.svg b/img/RDAS_logo_color.svg
old mode 100644
new mode 100755
diff --git a/requirements.txt b/requirements.txt
old mode 100644
new mode 100755
diff --git a/start_prod.py b/start_prod.py
old mode 100644
new mode 100755
diff --git a/sysvars.py b/sysvars.py
index 0fefab6..43cc129 100644
--- a/sysvars.py
+++ b/sysvars.py
@@ -41,10 +41,10 @@
db_abbrevs2 = {ct_db:'ct', pm_db:'pm', gnt_db:'gnt'}
# Paths to database creation and update source files
-ct_files_path = '{base_path}RDAS.CTKG/src/'.format(base_path=base_path)
-pm_files_path = '{base_path}RDAS.PAKG/src/'.format(base_path=base_path)
-gnt_files_path = '{base_path}RDAS.GFKG/src/'.format(base_path=base_path)
-gard_files_path = '{base_path}RDAS.GARD/src/'.format(base_path=base_path)
+ct_files_path = '{base_path}RDAS_CTKG/src/'.format(base_path=base_path)
+pm_files_path = '{base_path}RDAS_PAKG/src/'.format(base_path=base_path)
+gnt_files_path = '{base_path}RDAS_GFKG/src/'.format(base_path=base_path)
+gard_files_path = '{base_path}RDAS_GARD/src/'.format(base_path=base_path)
# Database names being used on the current server
convert = {ct_db:'trials', pm_db:'articles', gnt_db:'grants'}
diff --git a/transfer/README.md b/transfer/README.md
old mode 100644
new mode 100755