Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Move Date Range selection to Query Page #1396

Merged
merged 8 commits into from
Nov 13, 2023
Merged

Move Date Range selection to Query Page #1396

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
f927012
implementing date range in query page
BradleySappington Nov 7, 2023
3ee4483
fix some linting errors
BradleySappington Nov 7, 2023
5be0a6c
remove old date_range page
BradleySappington Nov 8, 2023
cac9cd2
pesky pep8 fixes
BradleySappington Nov 8, 2023
ea4ae52
more pep8
BradleySappington Nov 8, 2023
445b84b
fix anti-pattern
BradleySappington Nov 8, 2023
bafa6c2
put DateRange above GENERAL options row
BradleySappington Nov 8, 2023
092a544
Merge branch 'develop' into date_query
BradleySappington Nov 13, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1,163 changes: 813 additions & 350 deletions jwql/utils/constants.py
View file
Open in desktop

Large diffs are not rendered by default.

194 changes: 46 additions & 148 deletions jwql/website/apps/jwql/data_containers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@
import pandas as pd
import pyvo as vo
import requests
from datetime import datetime

from jwql.database import database_interface as di
from jwql.database.database_interface import load_connection
Expand All @@ -57,7 +58,7 @@
from jwql.utils.constants import EXPOSURE_PAGE_SUFFIX_ORDER, IGNORED_SUFFIXES, INSTRUMENT_SERVICE_MATCH
from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE, JWST_INSTRUMENT_NAMES
from jwql.utils.constants import REPORT_KEYS_PER_INSTRUMENT
from jwql.utils.constants import SUFFIXES_TO_ADD_ASSOCIATION, SUFFIXES_WITH_AVERAGED_INTS, QUERY_CONFIG_KEYS
from jwql.utils.constants import SUFFIXES_TO_ADD_ASSOCIATION, SUFFIXES_WITH_AVERAGED_INTS, QueryConfigKeys
from jwql.utils.credentials import get_mast_token
from jwql.utils.permissions import set_permissions
from jwql.utils.utils import get_rootnames_for_instrument_proposal
Expand Down Expand Up @@ -406,18 +407,18 @@ def get_available_suffixes(all_suffixes, return_untracked=True):
for poss_suffix in EXPOSURE_PAGE_SUFFIX_ORDER:
if 'crf' not in poss_suffix:
if (poss_suffix in all_suffixes
and poss_suffix not in suffixes):
suffixes.append(poss_suffix)
untracked_suffixes.remove(poss_suffix)
and poss_suffix not in suffixes):
suffixes.append(poss_suffix)
untracked_suffixes.remove(poss_suffix)
else:
# EXPOSURE_PAGE_SUFFIX_ORDER contains crf and crfints,
# but the actual suffixes in the data will be e.g. o001_crf,
# and there may be more than one crf file in the list of suffixes.
# So in this case, we strip the e.g. o001 from the
# suffixes and check which list elements match.
for image_suffix in all_suffixes:
if (image_suffix.endswith(poss_suffix)
and image_suffix not in suffixes):
if (image_suffix.endswith(poss_suffix) and
image_suffix not in suffixes):
suffixes.append(image_suffix)
untracked_suffixes.remove(image_suffix)

Expand Down Expand Up @@ -616,7 +617,8 @@ def get_edb_components(request):
mnemonic_query_result = get_mnemonic(mnemonic_identifier, start_time, end_time)

if len(mnemonic_query_result.data) == 0:
mnemonic_query_status = "QUERY RESULT RETURNED NO DATA FOR {} ON DATES {} - {}".format(mnemonic_identifier, start_time, end_time)
mnemonic_query_status = "QUERY RESULT RETURNED NO DATA FOR {} ON DATES {} - {}".format(mnemonic_identifier,
start_time, end_time)
else:
mnemonic_query_status = 'SUCCESS'

Expand Down Expand Up @@ -1248,7 +1250,8 @@ def get_instrument_proposals(instrument):
List of proposals for the given instrument
"""
tap_service = vo.dal.TAPService("https://vao.stsci.edu/caomtap/tapservice.aspx")
tap_results = tap_service.search(f"select distinct prpID from CaomObservation where collection='JWST' and maxLevel>0 and insName like '{instrument.lower()}%'")
tap_results = tap_service.search(f"""select distinct prpID from CaomObservation where collection='JWST'
and maxLevel>0 and insName like '{instrument.lower()}%'""")
prop_table = tap_results.to_table()
proposals = prop_table['prpID'].data
inst_proposals = sorted(proposals.compressed(), reverse=True)
Expand Down Expand Up @@ -1404,7 +1407,7 @@ def get_proposals_by_category(instrument):
unique_results = list(map(dict, set(tuple(sorted(sub.items())) for sub in results)))

# Make a dictionary of {program: category} to pull from
proposals_by_category = {d['program']:d['category'] for d in unique_results}
proposals_by_category = {d['program']: d['category'] for d in unique_results}

return proposals_by_category

Expand Down Expand Up @@ -1478,7 +1481,8 @@ def get_rootnames_for_proposal(proposal):
List of rootnames for the given instrument and proposal number
"""
tap_service = vo.dal.TAPService("https://vao.stsci.edu/caomtap/tapservice.aspx")
tap_results = tap_service.search(f"select observationID from dbo.CaomObservation where collection='JWST' and maxLevel=2 and prpID='{int(proposal)}'")
tap_results = tap_service.search(f"""select observationID from dbo.CaomObservation where
collection='JWST' and maxLevel=2 and prpID='{int(proposal)}'""")
prop_table = tap_results.to_table()
rootnames = prop_table['observationID'].data
return rootnames.compressed()
Expand All @@ -1500,32 +1504,51 @@ def get_rootnames_from_query(parameters):
"""

filtered_rootnames = []
DATE_FORMAT = "%Y/%m/%d %I:%M%p" # noqa n806

# Parse DATE_RANGE string into correct format
date_range = parameters[QueryConfigKeys.DATE_RANGE]
start_date_range, stop_date_range = date_range.split(" - ")
# Parse the strings into datetime objects
start_datetime = datetime.strptime(start_date_range, DATE_FORMAT)
stop_datetime = datetime.strptime(stop_date_range, DATE_FORMAT)
# store as astroquery Time objects in isot format to be used in filter (with mjd format)
start_time = Time(start_datetime.isoformat(), format="isot")
stop_time = Time(stop_datetime.isoformat(), format="isot")

# Each Query Selection is Instrument specific
for inst in parameters[QUERY_CONFIG_KEYS.INSTRUMENTS]:
for inst in parameters[QueryConfigKeys.INSTRUMENTS]:
# Make sure instruments are of the proper format for the archive query
inst = inst.lower()
current_ins_rootfileinfos = RootFileInfo.objects.filter(instrument=JWST_INSTRUMENT_NAMES_MIXEDCASE[inst])

# General fields
sort_type = parameters[QUERY_CONFIG_KEYS.SORT_TYPE]
look_status = parameters[QUERY_CONFIG_KEYS.LOOK_STATUS]
sort_type = parameters[QueryConfigKeys.SORT_TYPE]
look_status = parameters[QueryConfigKeys.LOOK_STATUS]

# Get a queryset of all observations STARTING within our date range
current_ins_rootfileinfos = current_ins_rootfileinfos.filter(
expstart__gte=start_time.mjd)
current_ins_rootfileinfos = current_ins_rootfileinfos.filter(
expstart__lte=stop_time.mjd)

if len(look_status) == 1:
viewed = (look_status[0] == 'VIEWED')
current_ins_rootfileinfos = current_ins_rootfileinfos.filter(viewed=viewed)
proposal_category = parameters[QUERY_CONFIG_KEYS.PROPOSAL_CATEGORY]
proposal_category = parameters[QueryConfigKeys.PROPOSAL_CATEGORY]
if len(proposal_category) > 0:
current_ins_rootfileinfos = current_ins_rootfileinfos.filter(obsnum__proposal__category__in=proposal_category)

# Instrument fields
inst_anomalies = parameters[QUERY_CONFIG_KEYS.ANOMALIES][inst]
inst_aperture = parameters[QUERY_CONFIG_KEYS.APERTURES][inst]
inst_detector = parameters[QUERY_CONFIG_KEYS.DETECTORS][inst]
inst_exp_type = parameters[QUERY_CONFIG_KEYS.EXP_TYPES][inst]
inst_filter = parameters[QUERY_CONFIG_KEYS.FILTERS][inst]
inst_grating = parameters[QUERY_CONFIG_KEYS.GRATINGS][inst]
inst_pupil = parameters[QUERY_CONFIG_KEYS.PUPILS][inst]
inst_read_patt = parameters[QUERY_CONFIG_KEYS.READ_PATTS][inst]
inst_subarray = parameters[QUERY_CONFIG_KEYS.SUBARRAYS][inst]
inst_anomalies = parameters[QueryConfigKeys.ANOMALIES][inst]
inst_aperture = parameters[QueryConfigKeys.APERTURES][inst]
inst_detector = parameters[QueryConfigKeys.DETECTORS][inst]
inst_exp_type = parameters[QueryConfigKeys.EXP_TYPES][inst]
inst_filter = parameters[QueryConfigKeys.FILTERS][inst]
inst_grating = parameters[QueryConfigKeys.GRATINGS][inst]
inst_pupil = parameters[QueryConfigKeys.PUPILS][inst]
inst_read_patt = parameters[QueryConfigKeys.READ_PATTS][inst]
inst_subarray = parameters[QueryConfigKeys.SUBARRAYS][inst]

if (inst_aperture != []):
current_ins_rootfileinfos = current_ins_rootfileinfos.filter(aperture__in=inst_aperture)
Expand Down Expand Up @@ -1980,131 +2003,6 @@ def thumbnails_ajax(inst, proposal, obs_num=None):
return data_dict


def thumbnails_date_range_ajax(inst, observations, inclusive_start_time_mjd, exclusive_stop_time_mjd):
"""Generate a page that provides data necessary to render thumbnails for
``archive_date_range`` template.

Parameters
----------
inst : str
Name of JWST instrument
observations: list
observation models to use to get filenames
inclusive_start_time_mjd : float
Start time in mjd format for date range
exclusive_stop_time_mjd : float
Stop time in mjd format for date range

Returns
-------
data_dict : dict
Dictionary of data needed for the ``thumbnails`` template
"""

data_dict = {'inst': inst,
'file_data': dict()}
exp_types = set()
exp_groups = set()

# Get the available files for the instrument
for observation in observations:
obs_num = observation.obsnum
proposal = observation.proposal.prop_id
filenames, columns = get_filenames_by_instrument(inst, proposal, observation_id=obs_num, other_columns=['expstart', 'exp_type'])
# Get set of unique rootnames
rootnames = set(['_'.join(f.split('/')[-1].split('_')[:-1]) for f in filenames])
# Gather data for each rootname, and construct a list of all observations in the proposal
for rootname in rootnames:
# Parse filename
try:
filename_dict = filename_parser(rootname)

# Weed out file types that are not supported by generate_preview_images
if 'stage_3' in filename_dict['filename_type']:
continue

except ValueError:
# Temporary workaround for noncompliant files in filesystem
filename_dict = {'activity': rootname[17:19],
'detector': rootname[26:],
'exposure_id': rootname[20:25],
'observation': rootname[7:10],
'parallel_seq_id': rootname[16],
'program_id': rootname[2:7],
'visit': rootname[10:13],
'visit_group': rootname[14:16],
'group_root': rootname[:26]}

# Get list of available filenames and exposure start times. All files with a given
# rootname will have the same exposure start time, so just keep the first.
available_files = []
exp_start = None
exp_type = None
for i, item in enumerate(filenames):
if rootname in item:
available_files.append(item)
if exp_start is None:
exp_start = columns['expstart'][i]
exp_type = columns['exp_type'][i]

if exp_start >= inclusive_start_time_mjd and exp_start < exclusive_stop_time_mjd:
exp_types.add(exp_type)
# Viewed is stored by rootname in the Model db. Save it with the data_dict
# THUMBNAIL_FILTER_LOOK is boolean accessed according to a viewed flag
try:
root_file_info = RootFileInfo.objects.get(root_name=rootname)
viewed = THUMBNAIL_FILTER_LOOK[root_file_info.viewed]
except RootFileInfo.DoesNotExist:
viewed = THUMBNAIL_FILTER_LOOK[0]

# Add to list of all exposure groups
exp_groups.add(filename_dict['group_root'])

# Add data to dictionary
data_dict['file_data'][rootname] = {}
data_dict['file_data'][rootname]['filename_dict'] = filename_dict
data_dict['file_data'][rootname]['available_files'] = available_files
data_dict['file_data'][rootname]["viewed"] = viewed
data_dict['file_data'][rootname]["exp_type"] = exp_type
data_dict['file_data'][rootname]['thumbnail'] = get_thumbnail_by_rootname(rootname)

try:
data_dict['file_data'][rootname]['expstart'] = exp_start
data_dict['file_data'][rootname]['expstart_iso'] = Time(exp_start, format='mjd').iso.split('.')[0]
except (ValueError, TypeError) as e:
logging.warning("Unable to populate exp_start info for {}".format(rootname))
logging.warning(e)
except KeyError:
print("KeyError with get_expstart for {}".format(rootname))

# Extract information for sorting with dropdown menus
# (Don't include the proposal as a sorting parameter if the proposal has already been specified)
detectors, proposals = [], []
for rootname in list(data_dict['file_data'].keys()):
proposals.append(data_dict['file_data'][rootname]['filename_dict']['program_id'])
try: # Some rootnames cannot parse out detectors
detectors.append(data_dict['file_data'][rootname]['filename_dict']['detector'])
except KeyError:
pass

dropdown_menus = {'detector': sorted(detectors),
'proposal': sorted(proposals),
'look': THUMBNAIL_FILTER_LOOK,
'exp_type': sorted(set(exp_types))}

data_dict['tools'] = MONITORS
data_dict['dropdown_menus'] = dropdown_menus

# Order dictionary by descending expstart time.
sorted_file_data = OrderedDict(sorted(data_dict['file_data'].items(),
key=lambda x: getitem(x[1], 'expstart'), reverse=True))

data_dict['file_data'] = sorted_file_data
data_dict['exp_groups'] = sorted(exp_groups)

return data_dict


def thumbnails_query_ajax(rootnames):
"""Generate a page that provides data necessary to render the
``thumbnails`` template.
Expand Down
20 changes: 13 additions & 7 deletions jwql/website/apps/jwql/forms.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,9 +59,10 @@ def view_function(request):
from jwql.website.apps.jwql.models import Anomalies


from jwql.utils.constants import (ANOMALY_CHOICES_PER_INSTRUMENT, ANOMALIES_PER_INSTRUMENT, APERTURES_PER_INSTRUMENT, DETECTOR_PER_INSTRUMENT, EXP_TYPE_PER_INSTRUMENT,
FILTERS_PER_INSTRUMENT, GENERIC_SUFFIX_TYPES, GRATING_PER_INSTRUMENT, GUIDER_FILENAME_TYPE, JWST_INSTRUMENT_NAMES_MIXEDCASE,
JWST_INSTRUMENT_NAMES_SHORTHAND, READPATT_PER_INSTRUMENT, IGNORED_SUFFIXES, SUBARRAYS_PER_INSTRUMENT, PUPILS_PER_INSTRUMENT,
from jwql.utils.constants import (ANOMALY_CHOICES_PER_INSTRUMENT, ANOMALIES_PER_INSTRUMENT, APERTURES_PER_INSTRUMENT, DETECTOR_PER_INSTRUMENT,
EXP_TYPE_PER_INSTRUMENT, FILTERS_PER_INSTRUMENT, GENERIC_SUFFIX_TYPES, GRATING_PER_INSTRUMENT,
GUIDER_FILENAME_TYPE, JWST_INSTRUMENT_NAMES_MIXEDCASE, JWST_INSTRUMENT_NAMES_SHORTHAND,
READPATT_PER_INSTRUMENT, IGNORED_SUFFIXES, SUBARRAYS_PER_INSTRUMENT, PUPILS_PER_INSTRUMENT,
LOOK_OPTIONS, SORT_OPTIONS, PROPOSAL_CATEGORIES)
from jwql.utils.utils import (get_config, get_rootnames_for_instrument_proposal, filename_parser, query_format)

Expand Down Expand Up @@ -144,6 +145,8 @@ class JwqlQueryForm(BaseForm):
look_status = forms.MultipleChoiceField(
required=False, choices=look_choices, widget=forms.CheckboxSelectMultiple)

date_range = forms.CharField(required=True)

cat_choices = [(query_format(choice), query_format(choice)) for choice in PROPOSAL_CATEGORIES]
proposal_category = forms.MultipleChoiceField(
required=False, choices=cat_choices, widget=forms.CheckboxSelectMultiple)
Expand Down Expand Up @@ -304,8 +307,10 @@ def clean_search(self):
# See if there are any matching proposals and, if so, what
# instrument they are for
proposal_string = '{:05d}'.format(int(search))
search_string_public = os.path.join(get_config()['filesystem'], 'public', 'jw{}'.format(proposal_string), '*', '*{}*.fits'.format(proposal_string))
search_string_proprietary = os.path.join(get_config()['filesystem'], 'proprietary', 'jw{}'.format(proposal_string), '*', '*{}*.fits'.format(proposal_string))
search_string_public = os.path.join(get_config()['filesystem'], 'public', 'jw{}'.format(proposal_string),
'*', '*{}*.fits'.format(proposal_string))
search_string_proprietary = os.path.join(get_config()['filesystem'], 'proprietary', 'jw{}'.format(proposal_string),
'*', '*{}*.fits'.format(proposal_string))
all_files = glob.glob(search_string_public)
all_files.extend(glob.glob(search_string_proprietary))

Expand Down Expand Up @@ -335,9 +340,10 @@ def clean_search(self):

if len(set(all_instruments)) > 1:
# Technically all proposal have multiple instruments if you include guider data. Remove Guider Data
instrument_routes = [format_html('<a href="/{}/archive/{}/obs{}">{}</a>', instrument, proposal_string[1:], all_observations[instrument][0], instrument) for instrument in set(all_instruments)]
instrument_routes = [format_html('<a href="/{}/archive/{}/obs{}">{}</a>', instrument, proposal_string[1:],
all_observations[instrument][0], instrument) for instrument in set(all_instruments)]
raise forms.ValidationError(
mark_safe(('Proposal contains multiple instruments, please click instrument link to view data: {}.').format(', '.join(instrument_routes)))) # nosec
mark_safe(('Proposal contains multiple instruments, please click instrument link to view data: {}.').format(', '.join(instrument_routes)))) # noqa

self.instrument = all_instruments[0]
else:
Expand Down
Loading