diff --git a/workflow/applications/applications.py b/workflow/applications/applications.py index 77c2ebafa3..0c81a5d722 100644 --- a/workflow/applications/applications.py +++ b/workflow/applications/applications.py @@ -31,78 +31,29 @@ def __init__(self, conf: Configuration) -> None: self.scheduler = Host().scheduler + # Get the most basic settings from config.base to determine + # experiment type ({NET}_{MODE}) base = conf.parse_config('config.base') self.mode = base['MODE'] - if self.mode not in self.VALID_MODES: raise NotImplementedError(f'{self.mode} is not a valid application mode.\n' f'Valid application modes are:\n' f'{", ".join(self.VALID_MODES)}\n') self.net = base['NET'] - self.do_atm = base.get('DO_ATM', True) - self.do_wave = base.get('DO_WAVE', False) - self.do_wave_bnd = base.get('DOBNDPNT_WAVE', False) - self.do_ocean = base.get('DO_OCN', False) - self.do_ice = base.get('DO_ICE', False) - self.do_aero = base.get('DO_AERO', False) - self.do_prep_obs_aero = base.get('DO_PREP_OBS_AERO', False) - self.do_bufrsnd = base.get('DO_BUFRSND', False) - self.do_gempak = base.get('DO_GEMPAK', False) - self.do_awips = base.get('DO_AWIPS', False) - self.do_verfozn = base.get('DO_VERFOZN', True) - self.do_verfrad = base.get('DO_VERFRAD', True) - self.do_vminmon = base.get('DO_VMINMON', True) - self.do_tracker = base.get('DO_TRACKER', True) - self.do_genesis = base.get('DO_GENESIS', True) - self.do_genesis_fsu = base.get('DO_GENESIS_FSU', False) - self.do_metp = base.get('DO_METP', False) - self.do_upp = not base.get('WRITE_DOPOST', True) - self.do_goes = base.get('DO_GOES', False) - self.do_mos = base.get('DO_MOS', False) - self.do_extractvars = base.get('DO_EXTRACTVARS', False) self.gfs_cyc = base.get('gfs_cyc') - self.do_hpssarch = base.get('HPSSARCH', False) - - self.nens = base.get('NMEM_ENS', 0) - self.fcst_segments = base.get('FCST_SEGMENTS', None) - - if not AppConfig.is_monotonic(self.fcst_segments): - raise ValueError(f'Forecast segments do not increase monotonically: {",".join(self.fcst_segments)}') - - self.wave_runs = None - if self.do_wave: - wave_run = base.get('WAVE_RUN', 'BOTH').lower() - if wave_run in ['both']: - self.wave_runs = ['gfs', 'gdas'] - elif wave_run in ['gfs', 'gdas']: - self.wave_runs = [wave_run] - - self.aero_anl_runs = None - self.aero_fcst_runs = None - if self.do_aero: - aero_anl_run = base.get('AERO_ANL_RUN', 'BOTH').lower() - if aero_anl_run in ['both']: - self.aero_anl_runs = ['gfs', 'gdas'] - elif aero_anl_run in ['gfs', 'gdas']: - self.aero_anl_runs = [aero_anl_run] - aero_fcst_run = base.get('AERO_FCST_RUN', None).lower() - if aero_fcst_run in ['both']: - self.aero_fcst_runs = ['gfs', 'gdas'] - elif aero_fcst_run in ['gfs', 'gdas']: - self.aero_fcst_runs = [aero_fcst_run] + print(f"Generating the XML for a {self.mode}_{self.net} case") def _init_finalize(self, conf: Configuration): print("Finalizing initialize") - # Get a list of all possible config files that would be part of the application - self.configs_names = self._get_app_configs() - # Get task names and runs for the application self.task_names = self.get_task_names() - self.runs = list(self.task_names.keys()) + + # Get a list of all possible config files that could be part of the application + self.configs_names = self._get_app_configs() # Initialize the configs and model_apps dictionaries self.model_apps = dict.fromkeys(self.runs) @@ -117,6 +68,103 @@ def _init_finalize(self, conf: Configuration): # Update the base config dictionary based on application and RUN self.configs[run]['base'] = self._update_base(self.configs[run]['base']) + def _get_run_options(self, conf: Configuration) -> Dict[str, Any]: + ''' + Determine the do_* and APP options for each RUN by sourcing config.base + for each RUN and collecting the flags into self.run_options + ''' + + run_options = dict.fromkeys(self.runs) + for run in self.runs: + #Read config.base with RUN specified + run_base = conf.parse_config('config.base', RUN=run) + + run_options[run]['do_wave_bnd'] = run_base.get('DOBNDPNT_WAVE', False) + run_options[run]['do_bufrsnd'] = run_base.get('DO_BUFRSND', False) + run_options[run]['do_gempak'] = run_base.get('DO_GEMPAK', False) + run_options[run]['do_awips'] = run_base.get('DO_AWIPS', False) + run_options[run]['do_verfozn'] = run_base.get('DO_VERFOZN', True) + run_options[run]['do_verfrad'] = run_base.get('DO_VERFRAD', True) + run_options[run]['do_vminmon'] = run_base.get('DO_VMINMON', True) + run_options[run]['do_tracker'] = run_base.get('DO_TRACKER', True) + run_options[run]['do_genesis'] = run_base.get('DO_GENESIS', True) + run_options[run]['do_genesis_fsu'] = run_base.get('DO_GENESIS_FSU', False) + run_options[run]['do_metp'] = run_base.get('DO_METP', False) + run_options[run]['do_upp'] = not run_base.get('WRITE_DOPOST', True) + run_options[run]['do_goes'] = run_base.get('DO_GOES', False) + run_options[run]['do_mos'] = run_base.get('DO_MOS', False) + run_options[run]['do_extractvars'] = run_base.get('DO_EXTRACTVARS', False) + + run_options[run]['do_atm'] = run_base.get('DO_ATM', True) + run_options[run]['do_wave'] = run_base.get('DO_WAVE', False) + run_options[run]['do_ocean'] = run_base.get('DO_OCN', False) + run_options[run]['do_ice'] = run_base.get('DO_ICE', False) + run_options[run]['do_aero'] = run_base.get('DO_AERO', False) + run_options[run]['do_prep_obs_aero'] = run_base.get('DO_PREP_OBS_AERO', False) + + run_options[run]['do_hpssarch'] = run_base.get('HPSSARCH', False) + run_options[run]['fcst_segments'] = run_base.get('FCST_SEGMENTS', None) + + if not AppConfig.is_monotonic(run_options[run]['fcst_segments']): + raise ValueError(f'Forecast segments do not increase monotonically: {",".join(self.fcst_segments)}') + + .wave_runs = None + if run_options[run]['do_wave']: + wave_run = run_base.get('WAVE_RUN', 'BOTH').lower() + if wave_run in ['both']: + wave_runs = ['gfs', 'gdas'] + elif wave_run in ['gfs', 'gdas']: + wave_runs = [wave_run] + + run_options[run]['do_wave'] = True if run in wave_runs else False + + aero_anl_runs = None + aero_fcst_runs = None + if run_options[run]['do_aero']: + aero_anl_run = run_base.get('AERO_ANL_RUN', 'BOTH').lower() + if aero_anl_run in ['both']: + aero_anl_runs = ['gfs', 'gdas'] + elif aero_anl_run in ['gfs', 'gdas']: + aero_anl_runs = [aero_anl_run] + + aero_fcst_run = base.get('AERO_FCST_RUN', None).lower() + if aero_fcst_run in ['both']: + aero_fcst_runs = ['gfs', 'gdas'] + elif aero_fcst_run in ['gfs', 'gdas']: + aero_fcst_runs = [aero_fcst_run] + + run_options[run]['do_aero_anl'] = True if run in aero_anl_runs else False + run_options[run]['do_aero_fcst'] = True if run in aero_fcst_runs else False + + # Append any MODE-specific options + run_options = self._netmode_run_options(run_base, run_options) + + # Return the dictionary of run options + return run_options + + @abstractmethod + def _netmode_run_options(self, base: Dict[str, Any], run_options: Dict[str, Any]) -> Dict[str, Any]: + ''' + Defines run-based options for a given NET_MODE case. + + Parameters + ---------- + base: Dict + Parsed config.base settings + + run_options: Dict + A dictionary with valid RUN-based sub-dictionaries containing generic options. + + Returns + ------- + run_options: Dict + Output dictionary with additional options valid for the given NET and MODE. + ''' + + # Valid NET_MODE options are defined in the appropriate subclass. + + pass + @abstractmethod def _get_app_configs(self): pass diff --git a/workflow/applications/gefs.py b/workflow/applications/gefs.py index afb4072596..c4f6a7feda 100644 --- a/workflow/applications/gefs.py +++ b/workflow/applications/gefs.py @@ -12,6 +12,7 @@ def __init__(self, conf: Configuration): base = conf.parse_config('config.base') self.run = base.get('RUN', 'gefs') + self.runs = [self.run] def _get_app_configs(self): """ diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index bbd342784c..93d523b585 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -11,25 +11,40 @@ class GFSCycledAppConfig(AppConfig): def __init__(self, conf: Configuration): super().__init__(conf) + # Re-read config.base without RUN specified to get the basic settings for + # cycled cases to be able to determine valid runs base = conf.parse_config('config.base') - self.do_hybvar = base.get('DOHYBVAR', False) - self.do_fit2obs = base.get('DO_FIT2OBS', True) - self.do_jediatmvar = base.get('DO_JEDIATMVAR', False) - self.do_jediatmens = base.get('DO_JEDIATMENS', False) - self.do_jediocnvar = base.get('DO_JEDIOCNVAR', False) - self.do_jedisnowda = base.get('DO_JEDISNOWDA', False) - self.do_mergensst = base.get('DO_MERGENSST', False) - self.do_vrfy_oceanda = base.get('DO_VRFY_OCEANDA', False) - - self.lobsdiag_forenkf = False - self.eupd_runs = None + self.runs = ["gdas"] + self.runs.append("gfs") if gfs_cyc > 0 else 0 + + self.ens_runs = None + if self.do_hybvar: - self.lobsdiag_forenkf = base.get('lobsdiag_forenkf', False) - eupd_run = base.get('EUPD_CYC', 'gdas').lower() - if eupd_run in ['both']: - self.eupd_runs = ['gfs', 'gdas'] - elif eupd_run in ['gfs', 'gdas']: - self.eupd_runs = [eupd_run] + ens_run = base.get('EUPD_CYC', 'gdas').lower() + if ens_run in ['both']: + self.ens_runs = ['gfs', 'gdas'] + elif ens_run in ['gfs', 'gdas']: + self.ens_runs = [ens_run] + + for ens_run in self.ens_runs: + self.runs.append(f"enkf{ens_run}") if ens_run in self.runs else 0 + + def _netmode_run_options(self, base: Dict[str, Any], run_options: Dict[str, Any]) -> Dict[str, Any]: + + run_options[run]['do_hybvar'] = base.get('DOHYBVAR', False) + run_options[run]['nens'] = base.get('NMEM_ENS', 0) + if run_options[run]['do_hybvar']: + run_options[run]['lobsdiag_forenkf'] = base.get('lobsdiag_forenkf', False) + + run_options[run]['do_fit2obs'] = base.get('DO_FIT2OBS', True) + run_options[run]['do_jediatmvar'] = base.get('DO_JEDIATMVAR', False) + run_options[run]['do_jediatmens'] = base.get('DO_JEDIATMENS', False) + run_options[run]['do_jediocnvar'] = base.get('DO_JEDIOCNVAR', False) + run_options[run]['do_jedisnowda'] = base.get('DO_JEDISNOWDA', False) + run_options[run]['do_mergensst'] = base.get('DO_MERGENSST', False) + run_options[run]['do_vrfy_oceanda'] = base.get('DO_VRFY_OCEANDA', False) + + return run_options def _get_app_configs(self): """ @@ -292,16 +307,16 @@ def get_task_names(self): tasks = dict() tasks['gdas'] = gdas_tasks - if self.do_hybvar and 'gdas' in self.eupd_runs: - enkfgdas_tasks = hybrid_tasks + hybrid_after_eupd_tasks + if self.do_hybvar and 'gdas' in self.ens_runs: + enkfgdas_tasks = hybrid_tasks + hybrid_after_ens_tasks tasks['enkfgdas'] = enkfgdas_tasks # Add RUN=gfs tasks if running early cycle if self.gfs_cyc > 0: tasks['gfs'] = gfs_tasks - if self.do_hybvar and 'gfs' in self.eupd_runs: - enkfgfs_tasks = hybrid_tasks + hybrid_after_eupd_tasks + if self.do_hybvar and 'gfs' in self.ens_runs: + enkfgfs_tasks = hybrid_tasks + hybrid_after_ens_tasks enkfgfs_tasks.remove("echgres") enkfgfs_tasks.remove("esnowrecen") tasks['enkfgfs'] = enkfgfs_tasks diff --git a/workflow/applications/gfs_forecast_only.py b/workflow/applications/gfs_forecast_only.py index 93551ac0cc..d0b1a9f82c 100644 --- a/workflow/applications/gfs_forecast_only.py +++ b/workflow/applications/gfs_forecast_only.py @@ -13,6 +13,7 @@ def __init__(self, conf: Configuration): base = conf.parse_config('config.base') self.aero_fcst_run = base.get('AERO_FCST_RUN', 'BOTH').lower() self.run = base.get('RUN', 'gfs') + self.runs = [self.run] self.exp_warm_start = base.get('EXP_WARM_START', False) def _get_app_configs(self): diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 8f2f612ea6..34b21fdf24 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -45,7 +45,7 @@ def prep(self): dump_path = self._template_to_rocoto_cycstring(self._base["COM_OBSDMP_TMPL"], {'DMPDIR': dmpdir, 'DUMP_SUFFIX': dump_suffix}) - gfs_enkf = True if self.app_config.do_hybvar and 'gfs' in self.app_config.eupd_runs else False + gfs_enkf = True if self.app_config.do_hybvar and 'gfs' in self.app_config.ens_runs else False deps = [] dep_dict = {'type': 'metatask', 'name': 'gdasatmos_prod', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} @@ -337,7 +337,7 @@ def atmanlinit(self): dependencies = rocoto.create_dependency(dep=deps) gfs_cyc = self._base["gfs_cyc"] - gfs_enkf = True if self.app_config.do_hybvar and 'gfs' in self.app_config.eupd_runs else False + gfs_enkf = True if self.app_config.do_hybvar and 'gfs' in self.app_config.ens_runs else False cycledef = self.run if self.run in ['gfs'] and gfs_enkf and gfs_cyc != 4: @@ -2686,8 +2686,6 @@ def _get_ecengroups(): def esfc(self): - # eupd_run = 'gdas' if 'gdas' in self.app_config.eupd_runs else 'gfs' - deps = [] dep_dict = {'type': 'task', 'name': f'{self.run.replace("enkf","")}analcalc'} deps.append(rocoto.add_dependency(dep_dict))