diff --git a/jwql/logging/logging_functions.py b/jwql/logging/logging_functions.py index 02fb94fc1..c5f0e6d71 100644 --- a/jwql/logging/logging_functions.py +++ b/jwql/logging/logging_functions.py @@ -66,7 +66,7 @@ def my_main_function(): from functools import wraps from jwql.permissions.permissions import set_permissions -from jwql.utils.utils import get_config +from jwql.utils.utils import get_config, ensure_dir_exists LOG_FILE_LOC = '' PRODUCTION_BOOL = '' @@ -144,6 +144,8 @@ def make_log_file(module, production_mode=True, path='./'): else: log_file = os.path.join(path, filename) + ensure_dir_exists(os.path.dirname(log_file)) + return log_file diff --git a/jwql/monitor_filesystem/monitor_filesystem.py b/jwql/monitor_filesystem/monitor_filesystem.py index 1c1b1d509..f798dc552 100755 --- a/jwql/monitor_filesystem/monitor_filesystem.py +++ b/jwql/monitor_filesystem/monitor_filesystem.py @@ -24,7 +24,6 @@ import statements: :: - from monitor_filesystem import filesystem_monitor from monitor_filesystem import plot_system_stats @@ -51,7 +50,7 @@ Notes ----- - The ``filesystem_monitor`` function queries the filesystem, + The ``monitor_filesystem`` function queries the filesystem, calculates the statistics and saves the output file(s) in the directory specified in the ``config.json`` file. @@ -67,12 +66,11 @@ import os import subprocess -from bokeh.plotting import figure, output_file, save +from bokeh.embed import components from bokeh.layouts import gridplot +from bokeh.plotting import figure, output_file, save -from jwql.logging.logging_functions import configure_logging -from jwql.logging.logging_functions import log_info -from jwql.logging.logging_functions import log_fail +from jwql.logging.logging_functions import configure_logging, log_info, log_fail from jwql.permissions.permissions import set_permissions from jwql.utils.utils import filename_parser from jwql.utils.utils import get_config @@ -80,11 +78,13 @@ @log_fail @log_info -def filesystem_monitor(): - """ Get statistics on filesystem""" +def monitor_filesystem(): + """Tabulates the inventory of the JWST filesystem, saving + statistics to files, and generates plots. + """ # Begin logging - logging.info("Beginning the script run: ") + logging.info('Beginning filesystem monitoring.') # Get path, directories and files in system and count files in all directories settings = get_config() @@ -95,6 +95,7 @@ def filesystem_monitor(): results_dict = defaultdict(int) size_dict = defaultdict(float) # Walk through all directories recursively and count files + logging.info('Searching filesystem...') for dirpath, dirs, files in os.walk(filesystem): results_dict['file_count'] += len(files) # find number of all files for filename in files: @@ -109,6 +110,7 @@ def filesystem_monitor(): instrument = detector[0:3] # first three characters of detector specify instrument results_dict[instrument] += 1 size_dict[instrument] += os.path.getsize(file_path) + logging.info('{} files found in filesystem'.format(results_dict['fits_files'])) # Get df style stats on file system out = subprocess.check_output('df {}'.format(filesystem), shell=True) @@ -130,6 +132,7 @@ def filesystem_monitor(): f.write("{0} {1:15d} {2:15d} {3:15d} {4:15d} {5}\n".format(now, results_dict['file_count'], total, available, used, percent_used)) set_permissions(statsfile) + logging.info('Saved file statistics to: {}'.format(statsfile)) # set up and read out stats on files by type filesbytype = os.path.join(outputs_dir, 'filesbytype.txt') @@ -139,6 +142,7 @@ def filesystem_monitor(): results_dict['rateints'], results_dict['i2d'], results_dict['nrc'], results_dict['nrs'], results_dict['nis'], results_dict['mir'], results_dict['gui'])) set_permissions(filesbytype, verbose=False) + logging.info('Saved file statistics by type to {}'.format(filesbytype)) # set up file size by type file sizebytype = os.path.join(outputs_dir, 'sizebytype.txt') @@ -148,6 +152,12 @@ def filesystem_monitor(): size_dict['rateints'], size_dict['i2d'], size_dict['nrc'], size_dict['nrs'], size_dict['nis'], size_dict['mir'], size_dict['gui'])) set_permissions(sizebytype, verbose=False) + logging.info('Saved file sizes by type to {}'.format(sizebytype)) + + logging.info('Filesystem statistics calculation complete.') + + # Create the plots + plot_system_stats(statsfile, filesbytype, sizebytype) def plot_system_stats(stats_file, filebytype, sizebytype): @@ -169,9 +179,10 @@ def plot_system_stats(stats_file, filebytype, sizebytype): outputs_dir = os.path.join(settings['outputs'], 'monitor_filesystem') # read in file of statistics - date, f_count, sysize, frsize, used, percent = np.loadtxt(os.path.join(outputs_dir, stats_file), dtype=str, unpack=True) - fits_files, uncalfiles, calfiles, ratefiles, rateintsfiles, i2dfiles, nrcfiles, nrsfiles, nisfiles, mirfiles, fgsfiles = np.loadtxt(os.path.join(outputs_dir, filebytype), dtype=str, unpack=True) - fits_sz, uncal_sz, cal_sz, rate_sz, rateints_sz, i2d_sz, nrc_sz, nrs_sz, nis_sz, mir_sz, fgs_sz = np.loadtxt(os.path.join(outputs_dir, sizebytype), dtype=str, unpack=True) + date, f_count, sysize, frsize, used, percent = np.loadtxt(stats_file, dtype=str, unpack=True) + fits_files, uncalfiles, calfiles, ratefiles, rateintsfiles, i2dfiles, nrcfiles, nrsfiles, nisfiles, mirfiles, fgsfiles = np.loadtxt(filebytype, dtype=str, unpack=True) + fits_sz, uncal_sz, cal_sz, rate_sz, rateints_sz, i2d_sz, nrc_sz, nrs_sz, nis_sz, mir_sz, fgs_sz = np.loadtxt(sizebytype, dtype=str, unpack=True) + logging.info('Read in file statistics from {}, {}, {}'.format(stats_file, filebytype, sizebytype)) # put in proper np array types and convert to GB sizes dates = np.array(date, dtype='datetime64') @@ -277,12 +288,36 @@ def plot_system_stats(stats_file, filebytype, sizebytype): p4.line(dates, fgs_size, legend='fgs fits files', line_color='darkred') p4.x(dates, fgs_size, color='darkred') - # create a layout with a grid pattern + # create a layout with a grid pattern to save all plots grid = gridplot([[p1, p2], [p3, p4]]) outfile = os.path.join(outputs_dir, "filesystem_monitor.html") output_file(outfile) save(grid) set_permissions(outfile) + logging.info('Saved plot of all statistics to {}'.format(outfile)) + + # Save each plot's components + plots = [p1, p2, p3, p4] + plot_names = ['filecount', 'system_stats', 'filecount_type', 'size_type'] + for plot, name in zip(plots, plot_names): + plot.sizing_mode = 'stretch_both' + script, div = components(plot) + + div_outfile = os.path.join(outputs_dir, "{}_component.html".format(name)) + with open(div_outfile, 'w') as f: + f.write(div) + f.close() + set_permissions(div_outfile) + + script_outfile = os.path.join(outputs_dir, "{}_component.js".format(name)) + with open(script_outfile, 'w') as f: + f.write(script) + f.close() + set_permissions(script_outfile) + + logging.info('Saved components files: {}_component.html and {}_component.js'.format(name, name)) + + logging.info('Filesystem statistics plotting complete.') # Begin logging: logging.info("Completed.") @@ -290,12 +325,8 @@ def plot_system_stats(stats_file, filebytype, sizebytype): if __name__ == '__main__': - inputfile = 'statsfile.txt' - filebytype = 'filesbytype.txt' - sizebytype = 'sizebytype.txt' - + # Configure logging module = os.path.basename(__file__).strip('.py') configure_logging(module) - filesystem_monitor() - plot_system_stats(inputfile, filebytype, sizebytype) + monitor_filesystem() diff --git a/jwql/monitor_mast/monitor_mast.py b/jwql/monitor_mast/monitor_mast.py index d74c7ec58..043b7f8c9 100644 --- a/jwql/monitor_mast/monitor_mast.py +++ b/jwql/monitor_mast/monitor_mast.py @@ -16,14 +16,17 @@ inventory, keywords = monitor_mast.jwst_inventory() """ +import logging import os from astroquery.mast import Mast from bokeh.charts import Donut, save, output_file +from bokeh.embed import components import pandas as pd -from ..permissions.permissions import set_permissions -from ..utils.utils import get_config, JWST_DATAPRODUCTS, JWST_INSTRUMENTS +from jwql.logging.logging_functions import configure_logging, log_info, log_fail +from jwql.permissions.permissions import set_permissions +from jwql.utils.utils import get_config, JWST_DATAPRODUCTS, JWST_INSTRUMENTS def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, @@ -157,6 +160,7 @@ def jwst_inventory(instruments=JWST_INSTRUMENTS, astropy.table.table.Table The table of record counts for each instrument and mode """ + logging.info('Searching database...') # Iterate through instruments inventory, keywords = [], {} for instrument in instruments: @@ -174,6 +178,9 @@ def jwst_inventory(instruments=JWST_INSTRUMENTS, # Add the keywords to the dict keywords[instrument] = instrument_keywords(instrument, caom=caom) + logging.info('Completed database search for {} instruments and {} data products.'. + format(instruments, dataproducts)) + # Make the table all_cols = ['instrument']+dataproducts+['total'] table = pd.DataFrame(inventory, columns=all_cols) @@ -185,20 +192,73 @@ def jwst_inventory(instruments=JWST_INSTRUMENTS, # Plot it if plot: + # Determine plot location and names + output_dir = get_config()['outputs'] + + if caom: + output_filename = 'database_monitor_caom' + else: + output_filename = 'database_monitor_jwst' # Make the plot plt = Donut(table, label=['instrument', 'dataproduct'], values='files', text_font_size='12pt', hover_text='files', name="JWST Inventory", plot_width=600, plot_height=600) - # Save the plot - if caom: - output_filename = 'database_monitor_caom.html' - else: - output_filename = 'database_monitor_jwst.html' - outfile = os.path.join(get_config()['outputs'], 'database_monitor', output_filename) + # Save the plot as full html + html_filename = output_filename + '.html' + outfile = os.path.join(output_dir, 'monitor_mast', html_filename) output_file(outfile) save(plt) set_permissions(outfile) + logging.info('Saved Bokeh plots as HTML file: {}'.format(html_filename)) + + # Save the plot as components + plt.sizing_mode = 'stretch_both' + script, div = components(plt) + + div_outfile = os.path.join(output_dir, 'monitor_mast', output_filename + "_component.html") + with open(div_outfile, 'w') as f: + f.write(div) + f.close() + set_permissions(div_outfile) + + script_outfile = os.path.join(output_dir, 'monitor_mast', output_filename + "_component.js") + with open(script_outfile, 'w') as f: + f.write(script) + f.close() + set_permissions(script_outfile) + + logging.info('Saved Bokeh components files: {}_component.html and {}_component.js'.format(output_filename, output_filename)) + return table, keywords + + +@log_fail +@log_info +def monitor_mast(): + """Tabulates the inventory of all JWST data products in the MAST + archive and generates plots. + """ + logging.info('Beginning database monitoring.') + + # Perform inventory of the JWST service + jwst_inventory(instruments=JWST_INSTRUMENTS, + dataproducts=['image', 'spectrum', 'cube'], + caom=False, plot=True) + + # Perform inventory of the CAOM service + jwst_inventory(instruments=JWST_INSTRUMENTS, + dataproducts=['image', 'spectrum', 'cube'], + caom=True, plot=True) + + +if __name__ == '__main__': + + # Configure logging + module = os.path.basename(__file__).strip('.py') + configure_logging(module) + + # Run the monitors + monitor_mast() diff --git a/jwql/utils/utils.py b/jwql/utils/utils.py index 7cba162db..3009e038a 100644 --- a/jwql/utils/utils.py +++ b/jwql/utils/utils.py @@ -25,6 +25,8 @@ import os import re +from ..permissions import permissions + __location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__))) JWST_INSTRUMENTS = sorted(['NIRISS', 'NIRCam', 'NIRSpec', 'MIRI', 'FGS']) @@ -46,6 +48,15 @@ 'Internal Lamp Monitor', 'Instrument Model Updates', 'Failed-open Shutter Monitor']} + +def ensure_dir_exists(fullpath): + """Creates dirs from ``fullpath`` if they do not already exist. + """ + if not os.path.exists(fullpath): + os.makedirs(fullpath) + permissions.set_permissions(fullpath) + + def get_config(): """Return a dictionary that holds the contents of the ``jwql`` config file. diff --git a/website/apps/jwql/data_containers.py b/website/apps/jwql/data_containers.py index 67f795cf6..477d597a4 100644 --- a/website/apps/jwql/data_containers.py +++ b/website/apps/jwql/data_containers.py @@ -80,10 +80,10 @@ def get_dashboard_components(): output_dir = get_config()['outputs'] name_dict = {'': '', - 'database_monitor': 'Database Monitor', + 'monitor_mast': 'Database Monitor', 'database_monitor_jwst': 'JWST', 'database_monitor_caom': 'JWST (CAOM)', - 'filesystem_monitor': 'Filesystem Monitor', + 'monitor_filesystem': 'Filesystem Monitor', 'filecount_type': 'Total File Counts by Type', 'size_type': 'Total File Sizes by Type', 'filecount': 'Total File Counts', diff --git a/website/apps/jwql/views.py b/website/apps/jwql/views.py index 12a4ea780..b778658b2 100644 --- a/website/apps/jwql/views.py +++ b/website/apps/jwql/views.py @@ -155,7 +155,7 @@ def dashboard(request): 'inst_list': JWST_INSTRUMENTS, 'tools': MONITORS, 'outputs': output_dir, - 'filesystem_html': os.path.join(output_dir, 'filesystem_monitor', 'filesystem_monitor.html'), + 'filesystem_html': os.path.join(output_dir, 'monitor_filesystem', 'filesystem_monitor.html'), 'dashboard_components': dashboard_components} return render(request, template, context)