-
Notifications
You must be signed in to change notification settings - Fork 7
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* Grant Pipeline-New GARD Mapping Process * Change get_node_counts * Removal of leftover merge text * More merge text removal * gfkg and pakg bug fixes * New CTKG pipeline * aact edit * changes for e alert * email alert chges 2 --------- Co-authored-by: Devon Joseph Leadman <[email protected]>
- Loading branch information
1 parent
130529c
commit a34b110
Showing
15 changed files
with
202 additions
and
53 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,41 @@ | ||
import os | ||
import sys | ||
workspace = os.path.dirname(os.path.abspath(__file__)) | ||
sys.path.append(workspace) | ||
sys.path.append('/home/leadmandj/RDAS/') | ||
sys.path.append(os.getcwd()) | ||
import sysvars | ||
from AlertCypher import AlertCypher | ||
from subprocess import * | ||
from time import sleep | ||
from datetime import datetime | ||
|
||
class Dump (): | ||
def __init__(self, mode='dev'): | ||
if mode in ['dev','test','prod']: | ||
self.mode = mode | ||
else: | ||
raise Exception | ||
|
||
self.db = AlertCypher('system') | ||
|
||
def dump_file (self, path, db_name): | ||
self.db.run(f'STOP DATABASE {db_name}') | ||
print(f'DATABASE {db_name} STOPPED') | ||
|
||
p = Popen(['sudo', '/opt/neo4j/bin/neo4j-admin', 'database', 'dump', f'{db_name}', f'--to-path={path}', '--overwrite-destination'], encoding='utf8') | ||
p.wait() | ||
print(f'DATABASE {db_name} DUMPED AT {path}') | ||
|
||
self.db.run(f'START DATABASE {db_name}') | ||
print(f'DATABASE {db_name} RESTARTED') | ||
|
||
def generate_backup_name (self, dump_name): | ||
cur_date = datetime.now().strftime("%m-%d-%y") | ||
return f'{self.mode}-{dump_name}-{cur_date}' | ||
|
||
def copy_to_backup(self, dump_name): | ||
filename = self.generate_backup_name(dump_name) | ||
p = Popen(['sudo', 'cp', f'{sysvars.transfer_path}{dump_name}.dump', f'{sysvars.backup_path}{dump_name}/{filename}.dump']) | ||
p.wait() | ||
print(f'DATABASE DUMP PUT INTO BACKUP FOLDER AT {sysvars.backup_path}{dump_name}.dump') |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,91 @@ | ||
import os | ||
import sys | ||
workspace = os.path.dirname(os.path.abspath(__file__)) | ||
sys.path.append(workspace) | ||
sys.path.append('/home/leadmandj/RDAS/') | ||
sys.path.append(os.getcwd()) | ||
import sysvars | ||
from AlertCypher import AlertCypher | ||
from subprocess import * | ||
from time import sleep | ||
|
||
class Transfer: | ||
def __init__ (self, mode=None): | ||
if mode in ['test','prod']: | ||
self.mode = mode | ||
else: | ||
raise Exception | ||
|
||
self.db = AlertCypher('system') | ||
|
||
def seed(self, dump_name, dump_folder): | ||
# dump_name = name of database (rdas.ctkg) | ||
# dump_folder = path to database files directory (RDAS/transfer/) | ||
try: | ||
self.db.run(f'DROP DATABASE {dump_name}') | ||
print('Dropped database...') | ||
|
||
p = Popen(['sudo', '/opt/neo4j/bin/neo4j-admin', 'database', 'load', f'{dump_name}', f'--from-path={dump_folder}', '--overwrite-destination'], encoding='utf8') | ||
p.wait() | ||
print('Waiting 20 seconds for database load') | ||
sleep(20) | ||
|
||
p = Popen(['sudo', '/opt/neo4j/bin/neo4j', 'restart'], encoding='utf-8') | ||
p.wait() | ||
print('Waiting 20 seconds for database restart') | ||
sleep(20) | ||
|
||
server_id = self.db.run(f"SHOW servers YIELD * WHERE name = \'{self.mode}1\' RETURN serverId").data()[0]['serverId'] | ||
print(f'SERVER ID LOCATED:: {server_id}') | ||
|
||
seed_query = f'CREATE DATABASE {dump_name} OPTIONS {{existingData: \'use\', existingDataSeedInstance: \'{server_id}\'}}' | ||
print(seed_query) | ||
print('Waiting 20 seconds for database seeding') | ||
sleep(20) | ||
|
||
except Exception as e: | ||
print('Did not drop or load database...') | ||
print(e) | ||
|
||
def detect(self, path): | ||
server = None | ||
config_title = None | ||
transfer_detection = {k:False for k in sysvars.dump_dirs} | ||
last_updates = {k:str() for k in sysvars.dump_dirs} | ||
|
||
if self.mode == 'test': | ||
server = 'TEST' | ||
config_title = 'TRANSFER' | ||
elif self.mode == 'prod': | ||
server = 'PROD' | ||
else: | ||
raise Exception | ||
|
||
if path == sysvars.transfer_path: | ||
config_title = 'TRANSFER' | ||
elif path == sysvars.approved_path: | ||
config_title = 'APPROVED' | ||
else: | ||
raise Exception | ||
|
||
for db_name in sysvars.dump_dirs: | ||
try: | ||
# Gets current last update timestamp from config | ||
last_mod_date = self.db.getConf(f'{server}_{config_title}_DETECTION', db_name) | ||
# stores value to return later | ||
last_updates[db_name] = last_mod_date | ||
print(f'{db_name} last mod:: {last_mod_date}') | ||
# Gets current modification timestamp of the targeted database dump file | ||
cur_mod_date = os.path.getmtime(f"{path}{db_name}.dump") | ||
print(f'{db_name} cur mod:: {cur_mod_date}') | ||
cur_mod_date = str(cur_mod_date) | ||
|
||
# If the config file last update and the current last modification date is different, detect as new transfer and update the config | ||
if not cur_mod_date == last_mod_date: | ||
transfer_detection[db_name] = True | ||
self.db.setConf(f'{server}_{config_title}_DETECTION', db_name, cur_mod_date) | ||
except Exception as e: | ||
print(e) | ||
transfer_detection[db_name] = False | ||
|
||
return [transfer_detection,last_updates] |
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -8,17 +8,13 @@ | |
# base_path = '/home/{current_user}/RDAS_master/{base_directory_name}/'.format(current_user=current_user, base_directory_name=base_directory_name) | ||
|
||
# RDAS Team contacts for emails | ||
contacts = [ | ||
"[email protected]" | ||
] | ||
contacts = { | ||
"dev": ["[email protected]"], | ||
"test": ["[email protected]","[email protected]"] | ||
} | ||
|
||
base_path = '/home/{current_user}/{base_directory_name}/'.format(current_user=current_user, base_directory_name=base_directory_name) | ||
|
||
# RDAS Team contacts for emails | ||
contacts = [ | ||
"[email protected]" | ||
] | ||
|
||
# Folder paths | ||
backup_path = '{base_path}backup/'.format(base_path=base_path) | ||
transfer_path = '{base_path}transfer/'.format(base_path=base_path) | ||
|
@@ -41,8 +37,8 @@ | |
gard_db = db_prefix+gard_db_name | ||
|
||
# Conversions | ||
dump_dirs = ['RDAS.CTKG','RDAS.PAKG','RDAS.GFKG','RDAS.GARD'] | ||
db_abbrevs = {'ct':'RDAS.CTKG', 'pm':'RDAS.PAKG', 'gnt':'RDAS.GFKG'} | ||
dump_dirs = ['rdas.ctkg','rdas.pakg','rdas.gfkg','rdas.gard'] | ||
db_abbrevs = {'ct':'rdas.ctkg', 'pm':'rdas.pakg', 'gnt':'rdas.gfkg'} | ||
db_abbrevs2 = {ct_db:'ct', pm_db:'pm', gnt_db:'gnt'} | ||
|
||
# Paths to database creation and update source files | ||
|