diff --git a/.gitignore b/.gitignore index d73dc34..6f9c8f4 100644 --- a/.gitignore +++ b/.gitignore @@ -43,4 +43,5 @@ migrated/*.dump approved/*.dump grant/raw/* grant/processed/* +crt/*.json removethis.py diff --git a/AlertCypher.py b/AlertCypher.py index ce4da69..ac66cfc 100644 --- a/AlertCypher.py +++ b/AlertCypher.py @@ -17,7 +17,7 @@ def __init__(self, db): user = os.environ['NEO4J_USERNAME'] password = os.environ['NEO4J_PASSWORD'] - watch("neo4j") + #watch("neo4j") connection = GraphDatabase.driver(uri=server_uri, auth=(user, password)) #encrypted=True, trusted_certificates=TrustAll() self.session = connection.session(database=db) diff --git a/crt/README.md b/crt/README.md new file mode 100644 index 0000000..0411b81 --- /dev/null +++ b/crt/README.md @@ -0,0 +1 @@ +# Location where the firebase access key will be stored diff --git a/email/alert.py b/email/alert.py index 174af8d..3b4f509 100644 --- a/email/alert.py +++ b/email/alert.py @@ -1,83 +1,57 @@ -#import django -#from django.core.mail import send_mail -#from django.template.loader import render_to_string -import os import boto3 from botocore.exceptions import ClientError -#os.environ['DJANGO_SETTINGS_MODULE'] = 'settings' -#django.setup() -''' -def clinical_msg (data): # dict - msg_plain = render_to_string('templates/clinical_notif.txt', data) - msg_html = render_to_string('templates/clinical_notif.html', data) - send_mail( - 'Test Email', - msg_plain, - 'from-email ', - ['RECIPIENT'], - html_message=msg_html - ) - -def pubmed_msg (data): # dict - msg_plain = render_to_string('templates/pubmed_notif.txt', data) - msg_html = render_to_string('templates/pubmed_notif.html', data) +sender_email = 'ncatsrdas@mail.nih.gov' - send_mail( - 'Test Email', - msg_plain, - 'from-email ', - ['RECIPIENT'], - html_message=msg_html +def setup_email_client(): + client = boto3.client( + service_name='ses', + region_name='us-east-1' ) + return client -def grant_msg (data): # dict - msg_plain = render_to_string('templates/grant_notif.txt', data) - msg_html = render_to_string('templates/grant_notif.html', data) - - send_mail( - 'Test Email', - msg_plain, - 'from-email ', - ['RECIPIENT'], - html_message=msg_html - ) +def send_email(sub,msg,recip,html=None,client=setup_email_client()): + if not client: + return -def test_msg (sub, msg, recip): - send_mail( - f'{sub}', - f'{msg}', - 'ncatsrdas@mail.nih.gov', - [f'{recip}'] + response = client.send_email( + Source=sender_email, + Destination={ + 'ToAddresses': [ + f'{recip}', + ], + }, + Message={ + 'Subject': { + 'Data': f'{sub}', + }, + 'Body': { + 'Text': { + 'Data': f'{msg}', + }, + 'Html': { + 'Data': f'{html}' + }, + } + } ) -''' - -def get_secret(): - secret_name = "/smtp/credentials" - region_name = "us-east-1" +def send_raw_email(sub,msg,recip,client=None): + if not client: + return - # Create a Secrets Manager client - session = boto3.session.Session() client = session.client( - service_name='secretsmanager', - region_name=region_name + service_name='ses', + region_name='us-east-1' + ) + + response = client.send_raw_email( + Source='ncatsrdas@mail.nih.gov', + Destinations=[ + f'{recip}', + ], + RawMessage={ + 'Data': f'To:{recip}\nFrom:ncatsrdas@mail.nih.gov\nSubject:{sub}\nMIME-Version: 1.0\nContent-type: Multipart/Mixed; boundary="NextPart"\n\n--NextPart\nContent-Type: text/plain\n\n{msg}\n\n' + }, ) - try: - get_secret_value_response = client.get_secret_value( - SecretId=secret_name - ) - except ClientError as e: - # For a list of exceptions thrown, see - # https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_GetSecretValue.html - raise e - - # Decrypts secret using the associated KMS key. - secret = get_secret_value_response['SecretString'] - print(secret) - - # Your code goes here. - -get_secret() -#test_msg('test','test','devon.leadman@nih.gov') diff --git a/email/send_single_alert_at_date.py b/email/send_single_alert_at_date.py deleted file mode 100644 index 4c6599c..0000000 --- a/email/send_single_alert_at_date.py +++ /dev/null @@ -1,36 +0,0 @@ -import django -from django.core.mail import send_mail -from django.template.loader import render_to_string -import os -import sys -workspace = os.path.dirname(os.path.abspath(__file__)) -sys.path.append(workspace) -sys.path.append(os.getcwd()) -os.environ['DJANGO_SETTINGS_MODULE'] = 'settings' -from AlertCypher import AlertCypher - -print('Insert name of database to get update from:') -dbname = input() -print('Insert desired alert date to select from (FORMAT: MM/DD/YY)') -date = input() - -selecteddb = AlertCypher(dbname) -nodename = 'Article' - -if dbname == 'clinical': - nodename = 'ClinicalTrial' -elif dbname == 'pubmed': - nodename = 'Article' -elif dbname == 'grant': - nodename = 'Project' - -response = selecteddb.run(f'MATCH (x:{nodename}) WHERE x.DateCreatedRDAS = \'{date}\' return count(x)').data()['count(x)'] - -send_mail ( - 'Test Email with Data', - f'Daily Digest for {date}\nThere are {response} new updates in our database', - 'ncatsrdas@mail.nih.gov', - ['shanmardev@hotmail.com'] -) - - diff --git a/email/ses_firebase.py b/email/ses_firebase.py index 144cc5f..9ccaff2 100644 --- a/email/ses_firebase.py +++ b/email/ses_firebase.py @@ -1,49 +1,107 @@ +import glob +import os +import sys +workspace = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(workspace) +sys.path.append(os.getcwd()) +import sysvars +from AlertCypher import AlertCypher from datetime import date import firebase_admin from firebase_admin import auth from firebase_admin import credentials from firebase_admin import firestore +import alert +def fill_template(type,data): + tabs = {'clinical':'trials', 'grant':'project', 'pubmed':'nonepi-articles'} + txt_db = {'clinical': 'clinical trial', 'grant': 'funded project', 'pubmed': 'publication'} + full_msg = '' + html = '' + + full_msg += 'You have {num} new entries for your subscribed rare diseases in the {db_title} database\nOut of that {num},\n\n'.format(num=data['total'], db_title=txt_db[type]) + + html += """ + + + +
+ Rare Disease Alert System +
+
+

{name}

+

On {date}, You have {num} new entries for your subscribed rare diseases in the {db_title} database

+

Out of that {num},

+
+

+ """.format(num=data['total'],images_path=sysvars.images_path,db_title=txt_db[type],name=data['name'],date=data['update_date']) + + for gard in data['subscriptions']: + if data[gard]['num'] > 0: + full_msg += '{name} [{ID}] - {num} new additions have been added to the database\n'.format(name=data[gard]['name'], num=data[gard]['num'], ID=gard) + html += """ + {name} [{ID}] - {num} new additions have been added to the database +
+ """.format(name=data[gard]['name'], num=data[gard]['num'], ID=gard, tab=tabs[type]) + + html += """ +

+

+ + + """ + return (full_msg,html) def send_mail(type, data): - if data['total'] > 0: - + print(data['total'],data['email']) + if data['total'] > 0 and data['email'] == 'zhuqianzq@gmail.com': + + data['email'] = 'devon.leadman@axleinfo.com' # TEST EMAIL + if type == "clinical": - print('To: {toemail}\nYou have {num} new entries for your subscribed rare diseases in the clinical trial database\nOut of that {num},\n'.format(toemail=data['email'],num=data['total'])) - for gard in data['subscriptions']: - if data[gard]['num'] > 0: - print('{name} [{ID}] - {num} new additions have been added to the database'.format(name=data[gard]['name'], num=data[gard]['num'], ID=gard)) - - #ses.clinical_msg() # pass in data for email as dict - elif type == "pubmed": - print('To: {toemail}\nYou have {num} new entries for your subscribed rare diseases'.format(toemail=data['email'],num=data['update_num'])) - #ses.pubmed_msg() - elif type == "grant": - print('To: {toemail}\nYou have {num} new entries for your subscribed rare diseases'.format(toemail=data['email'],num=data['update_num'])) - #ses.grant_msg() - - -def get_stats(db, type, gards): + txt,html = fill_template(type,data) + alert.send_email('RDAS-Alert: Clinical Trial update regarding your subscriptions', txt, data['email'], html=html) #data['email'] in place of email + print('Email Sent...') + + if type == "pubmed": + txt,html = fill_template(type,data) + alert.send_email('RDAS-Alert: Publication update regarding your subscriptions', txt, data['email'], html=html) + print('Email Sent...') + + if type == "grant": + txt,html = fill_template(type,data) + alert.send_email('RDAS-Alert: Funded Project update regarding your subscriptions', txt, data['email'], html=html) + print('Email Sent...') + +def get_stats(type, gards, date=None): + db = AlertCypher(type) return_data = dict() - now = date.today() - now = now.strftime("%m/%d/%y") - #now = "12/27/22" - convert = {'clinical':['ClinicalTrial','GARD','GARDId'], 'pubmed':['Article','Disease','gard_id'], 'grant':['Project','Disease','gard_id']} - response = db.run('MATCH (x:{node})--(y:{gardnode}) WHERE x.DateCreated = \"{now}\" AND y.{property} IN {list} RETURN COUNT(x)' - .format(node=convert[type][0], gardnode=convert[type][1], property=convert[type][2], list=list(gards.keys()), now=now)) + if date: + now = date + else: + now = date.today() + now = now.strftime("%m/%d/%y") + print(f'Searching for nodes created on {now}') + + convert = {'clinical':['ClinicalTrial','GARD','GardId'], 'pubmed':['Article','GARD','GardId'], 'grant':['Project','GARD','GardId']} + connect_to_gard = {'clinical':'--(:Condition)--(:Annotation)--','pubmed':'--','grant':'--'} + + response = db.run('MATCH (x:{node}){connection}(y:{gardnode}) WHERE x.DateCreatedRDAS = \"{now}\" AND y.{property} IN {list} RETURN COUNT(x)' + .format(node=convert[type][0], gardnode=convert[type][1], property=convert[type][2], list=list(gards.keys()), now=now, connection=connect_to_gard[type])) return_data['total'] = response.data()[0]['COUNT(x)'] for gard in gards.keys(): - response = db.run('MATCH (x:{node})--(y:{gardnode}) WHERE x.DateCreated = \"{now}\" AND y.{property} = \"{gard}\" RETURN COUNT(x)' - .format(node=convert[type][0], gardnode=convert[type][1], property=convert[type][2], gard=gard, now=now)) + response = db.run('MATCH (x:{node}){connection}(y:{gardnode}) WHERE x.DateCreatedRDAS = \"{now}\" AND y.{property} = \"{gard}\" RETURN COUNT(x)' + .format(node=convert[type][0], gardnode=convert[type][1], property=convert[type][2], gard=gard, now=now, connection=connect_to_gard[type])) return_data[gard] = {'name':gards[gard],'num':response.data()[0]['COUNT(x)']} + return_data['update_date'] = now return return_data -def trigger_email(db, type): +def trigger_email(type,date=None): convert = {'clinical':'trials', 'pubmed':'articles', 'grant':'grants'} user_data = dict() - cred = credentials.Certificate('new\\firestore\\key.json') + cred = credentials.Certificate(sysvars.firebase_key_path) firebase_admin.initialize_app(cred) firestore_db = firestore.client() @@ -55,7 +113,6 @@ def trigger_email(db, type): else: print(u'Document Doesnt Exist') - for firestore_user, data in user_data.items(): subscript_gard = dict() for subscript in data['subscriptions']: @@ -63,15 +120,18 @@ def trigger_email(db, type): if convert[type] in subscript['alerts']: subscript_gard[subscript['gardID']] = subscript['diseaseName'] except KeyError: + print('') pass - users = auth.list_users() if users: users = users.iterate_all() for user in users: uid = user.uid - if uid == firestore_user: - update_data = get_stats(db, type, subscript_gard) + if uid == firestore_user and len(subscript_gard) > 0: + update_data = get_stats(type, subscript_gard, date) update_data['email'] = user.email + update_data['name'] = user_data[uid]['displayName'] update_data['subscriptions'] = list(subscript_gard.keys()) send_mail(type, update_data) + +trigger_email(sysvars.gnt_db, '04/27/23') diff --git a/email/settings.py b/email/settings.py deleted file mode 100644 index d2c59e1..0000000 --- a/email/settings.py +++ /dev/null @@ -1,7 +0,0 @@ -import os - -DEBUG = True -EMAIL_BACKEND = 'django_ses.SESBackend' -AWS_ACCESS_KEY_ID = os.environ['AWS_ACCESS_KEY_ID'] -AWS_SECRET_ACCESS_KEY = os.environ['AWS_SECRET_ACCESS_KEY'] -AWS_SESSION_TOKEN = os.environ['AWS_SESSION_TOKEN'] diff --git a/email/templates/clinical_notif.html b/email/templates/clinical_notif.html deleted file mode 100644 index 079e40f..0000000 --- a/email/templates/clinical_notif.html +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - - - - - -

Hello {{name}},

-

{{num}} Clinical Trials have been added to RDAS

-

-Rare Disease Alert System

- - \ No newline at end of file diff --git a/email/templates/clinical_notif.txt b/email/templates/clinical_notif.txt deleted file mode 100644 index 12709e1..0000000 --- a/email/templates/clinical_notif.txt +++ /dev/null @@ -1,3 +0,0 @@ -Hello {{name}}, -{{num}} Clinical Trials have just been added to RDAS --Rare Disease Alert System \ No newline at end of file diff --git a/email/templates/grant_notif.html b/email/templates/grant_notif.html deleted file mode 100644 index c595e3e..0000000 --- a/email/templates/grant_notif.html +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - - - - - -

Hello {{name}},

-

{{num}} NIH grants have been added to RDAS

-

-Rare Disease Alert System

- - \ No newline at end of file diff --git a/email/templates/grant_notif.txt b/email/templates/grant_notif.txt deleted file mode 100644 index baf2d35..0000000 --- a/email/templates/grant_notif.txt +++ /dev/null @@ -1,3 +0,0 @@ -Hello {{name}}, -{{num}} NIH grants have just been added to RDAS --Rare Disease Alert System \ No newline at end of file diff --git a/email/templates/pubmed_notif.html b/email/templates/pubmed_notif.html deleted file mode 100644 index 05aa836..0000000 --- a/email/templates/pubmed_notif.html +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - - - - - -

Hello {{name}},

-

{{num}} Pubmed articles have been added to RDAS

-

-Rare Disease Alert System

- - \ No newline at end of file diff --git a/email/templates/pubmed_notif.txt b/email/templates/pubmed_notif.txt deleted file mode 100644 index 792e30e..0000000 --- a/email/templates/pubmed_notif.txt +++ /dev/null @@ -1,3 +0,0 @@ -Hello {{name}}, -{{num}} PubMed articles have just been added to RDAS --Rare Disease Alert System \ No newline at end of file diff --git a/email/testaws.py b/email/testaws.py deleted file mode 100644 index cfe42d8..0000000 --- a/email/testaws.py +++ /dev/null @@ -1,22 +0,0 @@ -import boto3 -from botocore.exceptions import ClientError - -def send_raw_email(sub,msg.recip): - client = session.client( - service_name='ses', - region_name='us-east-1' - ) - ses_client = boto3.client("ses", region_name="us-east-1") - CHARSET = "UTF-8" - - response = client.send_raw_email( - Source='ncatsrdas@mail.nih.gov', - Destinations=[ - f'{recip}', - ], - RawMessage={ - 'Data': f'To:{recip}\nFrom:ncatsrdas@mail.nih.gov\nSubject:{sub}\nMIME-Version: 1.0\nContent-type: Multipart/Mixed; boundary="NextPart"\n\n--NextPart\nContent-Type: text/plain\n\n{msg}\n\n' - }, - ) - -send_raw_email('Test Email', 'This is a test email with formatting', 'devon.leadman@axleinfo.com') diff --git a/img/RDAS_logo_color.png b/img/RDAS_logo_color.png new file mode 100644 index 0000000..5f07699 Binary files /dev/null and b/img/RDAS_logo_color.png differ diff --git a/img/RDAS_logo_color.svg b/img/RDAS_logo_color.svg new file mode 100644 index 0000000..034f042 --- /dev/null +++ b/img/RDAS_logo_color.svg @@ -0,0 +1,90 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/sysvars.py b/sysvars.py index 19f618a..16bab95 100644 --- a/sysvars.py +++ b/sysvars.py @@ -2,28 +2,36 @@ current_version = 1.4 +# Basic user information current_user = 'leadmandj' base_directory_name = 'RDAS' base_path = '/home/{current_user}/{base_directory_name}/'.format(current_user=current_user, base_directory_name=base_directory_name) +# Folder paths backup_path = '{base_path}backup/'.format(base_path=base_path) transfer_path = '{base_path}transfer/'.format(base_path=base_path) migrated_path = '{base_path}migrated/'.format(base_path=base_path) approved_path = '{base_path}approved/'.format(base_path=base_path) +images_path = '{base_path}img/'.format(base_path=base_path) +firebase_key_path = '{base_path}crt/ncats-summer-interns-firebase-adminsdk-9g7zz-a4e783d24c.json'.format(base_path=base_path) +# Conversions dump_dirs = ['clinical','pubmed','grant','gard'] db_abbrevs = {'ct':'clinical', 'pm':'pubmed', 'gnt':'grant'} +# Paths to database creation and update source files ct_files_path = '{base_path}/clinical/src/'.format(base_path=base_path) pm_files_path = '{base_path}/pubmed/src/'.format(base_path=base_path) gnt_files_path = '{base_path}/grant/src/'.format(base_path=base_path) gard_files_path = '{base_path}/gard/src/'.format(base_path=base_path) +# Database names being used on the current server ct_db = 'clinical' pm_db = 'pubmed' gnt_db = 'grant' gard_db = 'gard' +# Server URLS and addresses epiapi_url = "http://ncats-rdas-lnx-dev.ncats.nih.gov:80/api/" rdas_urls = {'dev':'rdas-dev.ncats.nih.gov','test':"ncats-neo4j-lnx-test1.ncats.nih.gov",'prod':"ncats-neo4j-lnx-prod1.ncats.nih.gov"}