From 6577584df45ec6ab283433a89e362618fbe2ae53 Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA <58948505+AndrewEichmann-NOAA@users.noreply.github.com> Date: Wed, 10 Jan 2024 13:31:11 -0500 Subject: [PATCH 1/4] Remove tests with R2D2 and anything referencing it (#836) Removes soca ctests related to R2D2, associated data, and any lines in scripts, yamls, or other config files referencing it. SOCA ctests now run on obs prepped by the prep obs task. Almost takes care of https://github.com/NOAA-EMC/GDASApp/issues/815 as references are still present in global-workflow task config files. `logging.info` has been replaced by `print` because logging hasn't been working --------- Co-authored-by: Guillaume Vernieres --- .github/workflows/unittests.yaml | 9 - modulefiles/GDAS/hera.lua | 1 - modulefiles/GDAS/orion.lua | 1 - parm/soca/obs/obs_list_small.yaml | 1 + parm/soca/obsproc/obs_list_obsproc.yaml | 2 +- parm/templates/stage.yaml | 7 - scripts/exgdas_global_marine_analysis_prep.py | 13 +- scripts/exglobal_prep_ocean_obs.py | 3 + test/atm/CMakeLists.txt | 10 - test/atm/test_store_gsi_satbias.py | 74 ---- test/soca/CMakeLists.txt | 11 - test/soca/create_obsdb.py | 82 ---- test/soca/gw/CMakeLists.txt | 13 +- test/soca/gw/run_jjobs.yaml.test | 3 - test/soca/gw/test_concatioda.py | 29 -- test/soca/testdata/adt.nc.cdl | 139 ------- test/soca/testdata/icec.nc.cdl | 139 ------- test/soca/testdata/prof.nc.cdl | 388 ------------------ test/soca/testdata/sss.nc.cdl | 153 ------- test/soca/testdata/sst.nc.cdl | 203 --------- ush/examples/run_scripts/soca025.yaml | 3 - ush/r2d2/example_store_bkg.py | 46 --- ush/r2d2/proc_ops2r2d2.sh | 279 ------------- ush/r2d2/r2d2_store.py | 12 - ush/soca/examples/run_jjobs_hera.yaml | 2 - ush/soca/examples/run_jjobs_hera_025.yaml | 2 - ush/ufsda/__init__.py | 3 - ush/ufsda/archive.py | 62 --- ush/ufsda/post.py | 73 ---- ush/ufsda/r2d2.py | 115 ------ ush/ufsda/stage.py | 239 ----------- 31 files changed, 9 insertions(+), 2108 deletions(-) delete mode 100755 test/atm/test_store_gsi_satbias.py delete mode 100755 test/soca/create_obsdb.py delete mode 100755 test/soca/gw/test_concatioda.py delete mode 100644 test/soca/testdata/adt.nc.cdl delete mode 100644 test/soca/testdata/icec.nc.cdl delete mode 100644 test/soca/testdata/prof.nc.cdl delete mode 100644 test/soca/testdata/sss.nc.cdl delete mode 100644 test/soca/testdata/sst.nc.cdl delete mode 100644 ush/r2d2/example_store_bkg.py delete mode 100755 ush/r2d2/proc_ops2r2d2.sh delete mode 100755 ush/r2d2/r2d2_store.py delete mode 100644 ush/ufsda/archive.py delete mode 100644 ush/ufsda/post.py delete mode 100644 ush/ufsda/r2d2.py diff --git a/.github/workflows/unittests.yaml b/.github/workflows/unittests.yaml index 2dba08de5..c103aee50 100644 --- a/.github/workflows/unittests.yaml +++ b/.github/workflows/unittests.yaml @@ -15,15 +15,6 @@ jobs: pip install netCDF4 pip install xarray - - name: Install other dependencies - run: | - wget https://ftp.emc.ncep.noaa.gov/static_files/public/GDASApp/r2d2-bb361c2.tar.gz - wget https://ftp.emc.ncep.noaa.gov/static_files/public/GDASApp/solo-877b414.tar.gz - tar -xvf r2d2-bb361c2.tar.gz - tar -xvf solo-877b414.tar.gz - cd solo && pip install . && cd .. - cd r2d2 && pip install . && cd .. - - name: Checkout wxflow uses: actions/checkout@v3 with: diff --git a/modulefiles/GDAS/hera.lua b/modulefiles/GDAS/hera.lua index e1014f92d..c88e64040 100644 --- a/modulefiles/GDAS/hera.lua +++ b/modulefiles/GDAS/hera.lua @@ -79,7 +79,6 @@ local mpinproc = '-n' setenv('MPIEXEC_EXEC', mpiexec) setenv('MPIEXEC_NPROC', mpinproc) -setenv('R2D2_CONFIG', '/scratch1/NCEPDEV/stmp4/Cory.R.Martin/R2D2_SHARED/config_hera.yaml') setenv("CRTM_FIX","/scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/fix/crtm/2.4.0") setenv("GDASAPP_TESTDATA","/scratch1/NCEPDEV/da/Cory.R.Martin/CI/GDASApp/data") prepend_path("PATH","/scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/intel-18.0.5.274/prod_util/1.2.2/bin") diff --git a/modulefiles/GDAS/orion.lua b/modulefiles/GDAS/orion.lua index 00ddecce0..0625ddc4c 100644 --- a/modulefiles/GDAS/orion.lua +++ b/modulefiles/GDAS/orion.lua @@ -78,7 +78,6 @@ local mpinproc = '-n' setenv('MPIEXEC_EXEC', mpiexec) setenv('MPIEXEC_NPROC', mpinproc) -setenv('R2D2_CONFIG', '/work2/noaa/da/cmartin/GDASApp/R2D2_SHARED/config_orion.yaml') setenv("CRTM_FIX","/work2/noaa/da/cmartin/GDASApp/fix/crtm/2.4.0") setenv("GDASAPP_TESTDATA","/work2/noaa/da/cmartin/CI/GDASApp/data") prepend_path("PATH","/apps/contrib/NCEP/libs/hpc-stack/intel-2018.4/prod_util/1.2.2/bin") diff --git a/parm/soca/obs/obs_list_small.yaml b/parm/soca/obs/obs_list_small.yaml index b9d9e24a4..4ff437200 100644 --- a/parm/soca/obs/obs_list_small.yaml +++ b/parm/soca/obs/obs_list_small.yaml @@ -4,5 +4,6 @@ observers: - !INC ${OBS_YAML_DIR}/adt_j2.yaml #- !INC ${OBS_YAML_DIR}/salt_profile_fnmoc.yaml - !INC ${OBS_YAML_DIR}/sss_smap.yaml +- !INC ${OBS_YAML_DIR}/sst_metopb_l3u_so025.yaml - !INC ${OBS_YAML_DIR}/sst_noaa19_l3u.yaml - !INC ${OBS_YAML_DIR}/icec_emc.yaml diff --git a/parm/soca/obsproc/obs_list_obsproc.yaml b/parm/soca/obsproc/obs_list_obsproc.yaml index e37c2f83b..7a884f9ea 100644 --- a/parm/soca/obsproc/obs_list_obsproc.yaml +++ b/parm/soca/obsproc/obs_list_obsproc.yaml @@ -5,7 +5,7 @@ observers: #- !INC ${OBS_YAML_DIR}/salt_profile_fnmoc.yaml - !INC ${OBS_YAML_DIR}/sss_smap.yaml - !INC ${OBS_YAML_DIR}/sss_smos.yaml -- !INC ${OBS_YAML_DIR}/sst_noaa19_l3u.yaml +#- !INC ${OBS_YAML_DIR}/sst_noaa19_l3u.yaml - !INC ${OBS_YAML_DIR}/sst_metopb_l3u_so025.yaml - !INC ${OBS_YAML_DIR}/icec_emc.yaml - !INC ${OBS_YAML_DIR}/adt_3b_egm2008.yaml diff --git a/parm/templates/stage.yaml b/parm/templates/stage.yaml index 9b58b6070..4ebfc5b1c 100644 --- a/parm/templates/stage.yaml +++ b/parm/templates/stage.yaml @@ -7,13 +7,6 @@ COMOUT: !ENV ${COMOUT} DATA: !ENV ${DATA} background_time: '{{background_time}}' dump: !ENV ${CDUMP} -r2d2_obs_db: !ENV ${R2D2_OBS_DB} -r2d2_obs_dump: !ENV ${R2D2_OBS_DUMP} -r2d2_obs_src: !ENV ${R2D2_OBS_SRC} -r2d2_bc_src: !ENV ${R2D2_BC_SRC} -r2d2_bc_dump: !ENV ${R2D2_BC_DUMP} -r2d2 window length: !ENV ${R2D2_OBS_WINDOW} -r2d2 window begin: '{{r2d2_window_begin}}' fv3jedi_stage_files: !ENV ${FV3JEDI_STAGE_YAML} fv3jedi_stage: !ENV ${FV3JEDI_STAGE_YAML} stage_dir: !ENV ${DATA} diff --git a/scripts/exgdas_global_marine_analysis_prep.py b/scripts/exgdas_global_marine_analysis_prep.py index b5e7117fc..4af8d0189 100755 --- a/scripts/exgdas_global_marine_analysis_prep.py +++ b/scripts/exgdas_global_marine_analysis_prep.py @@ -45,7 +45,7 @@ # import UFSDA utilities import ufsda -from ufsda.stage import obs, soca_fix +from ufsda.stage import soca_fix def agg_seaice(fname_in, fname_out): @@ -259,24 +259,14 @@ def find_clim_ens(input_date): logging.info(f"---------------- Stage observations") -# setup the archive, local and shared R2D2 databases -ufsda.r2d2.setup(r2d2_config_yaml=os.path.join(anl_dir, 'r2d2_config.yaml'), shared_root=comin_obs) - # create config dict from runtime env envconfig = {'window_begin': f"{window_begin.strftime('%Y-%m-%dT%H:%M:%SZ')}", - 'r2d2_obs_src': os.getenv('R2D2_OBS_SRC'), - 'r2d2_obs_dump': os.getenv('R2D2_OBS_DUMP'), - 'r2d2_obs_db': os.getenv('R2D2_OBS_DB'), 'ATM_WINDOW_BEGIN': window_begin_iso, 'ATM_WINDOW_MIDDLE': window_middle_iso, 'ATM_WINDOW_LENGTH': f"PT{os.getenv('assim_freq')}H"} stage_cfg = YAMLFile(path=os.path.join(gdas_home, 'parm', 'templates', 'stage.yaml')) stage_cfg = Template.substitute_structure(stage_cfg, TemplateConstants.DOUBLE_CURLY_BRACES, envconfig.get) stage_cfg = Template.substitute_structure(stage_cfg, TemplateConstants.DOLLAR_PARENTHESES, envconfig.get) -stage_cfg['r2d2_obs_out'] = os.getenv('COM_OBS') - -# stage observations from R2D2 COMIN_OBS to COM_OBS -ufsda.stage.obs(stage_cfg) # concatenate altimeters into one obs space # TODO (SAMG)temporary, move this into the obs procecing eventually @@ -294,6 +284,7 @@ def find_clim_ens(input_date): logging.info(f"******* {obs_file}") obs_src = os.path.join(os.getenv('COM_OBS'), obs_file) obs_dst = os.path.join(os.path.abspath(obs_in), obs_file) + logging.info(f"******* {obs_src}") if os.path.exists(obs_src): logging.info(f"******* fetching {obs_file}") obs_list.append([obs_src, obs_dst]) diff --git a/scripts/exglobal_prep_ocean_obs.py b/scripts/exglobal_prep_ocean_obs.py index 1d9bd81f0..c35689ead 100755 --- a/scripts/exglobal_prep_ocean_obs.py +++ b/scripts/exglobal_prep_ocean_obs.py @@ -85,4 +85,7 @@ print("CRITICAL: Ill-formed OBS_YAML file, exiting") raise +if not os.path.exists(COMOUT_OBS): + os.makedirs(COMOUT_OBS) + FileHandler({'copy': filesToSave}).sync() diff --git a/test/atm/CMakeLists.txt b/test/atm/CMakeLists.txt index 117e2ea8e..6dd33d72d 100644 --- a/test/atm/CMakeLists.txt +++ b/test/atm/CMakeLists.txt @@ -15,16 +15,6 @@ if (BUILD_GDASBUNDLE) COMMAND ${PROJECT_SOURCE_DIR}/test/atm/test_convert_gsi_satbias.sh ${PROJECT_SOURCE_DIR} ${PROJECT_BINARY_DIR} WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/) - # test saving the output of the previous test to an R2D2 database - add_test(NAME test_gdasapp_store_gsi_satbias - COMMAND ${PROJECT_SOURCE_DIR}/test/atm/test_store_gsi_satbias.py - WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/) - set_tests_properties(test_gdasapp_store_gsi_satbias - PROPERTIES - DEPENDS "test_gdasapp_convert_gsi_satbias" - ENVIRONMENT "PYTHONPATH=${PROJECT_BINARY_DIR}/ush:$ENV{PYTHONPATH}" - ) - # Test exgdas scripts from the global-worflow if (WORKFLOW_TESTS) add_subdirectory(global-workflow) diff --git a/test/atm/test_store_gsi_satbias.py b/test/atm/test_store_gsi_satbias.py deleted file mode 100755 index 84350adae..000000000 --- a/test/atm/test_store_gsi_satbias.py +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env python3 -import os -from r2d2 import store -from solo.configuration import Configuration -from solo.date import date_sequence -import yaml -import ufsda.r2d2 - - -def store_satbias(yaml_file): - config = Configuration(yaml_file) - ufsda.r2d2.store(config) - - -if __name__ == "__main__": - # create a dummy R2D2 config YAML - r2d2_config = { - 'databases': - {'test': {'cache_fetch': False, - 'class': 'LocalDB', - 'root': os.path.join(os.getcwd(), 'r2d2-test')}, }, - 'fetch_order': ['test'], - 'store_order': ['test'], - } - with open(os.path.join(os.getcwd(), 'testinput', 'r2d2_config_test.yaml'), 'w') as f: - yaml.dump(r2d2_config, f, sort_keys=False, default_flow_style=False) - os.environ['R2D2_CONFIG'] = os.path.join(os.getcwd(), 'testinput', 'r2d2_config_test.yaml') - # create YAML for satbias - satbias = { - 'start': '2022-04-01T00:00:00Z', - 'end': '2022-04-01T00:00:00Z', - 'step': 'PT6H', - 'source_dir': os.path.join(os.getcwd(), 'testoutput', 'satbias'), - 'source_file_fmt': '{source_dir}/{dump}.{year}{month}{day}/{hour}/atmos/{obs_type}_satbias.nc4', - 'type': 'bc', - 'file_type': 'satbias', - 'database': 'test', - 'provider': 'gsi', - 'experiment': 'gdasapp', - 'obs_types': [ - 'amsua_metop-b', - 'atms_npp', - 'cris-fsr_npp', - 'iasi_metop-c', - ], - } - satbias_yaml = os.path.join(os.getcwd(), 'testinput', 'r2d2_store_satbias.yaml') - with open(satbias_yaml, 'w') as f: - yaml.dump(satbias, f, sort_keys=False, default_flow_style=False) - # create YAML for tlapse - tlapse = { - 'start': '2022-04-01T00:00:00Z', - 'end': '2022-04-01T00:00:00Z', - 'step': 'PT6H', - 'source_dir': os.path.join(os.getcwd(), 'testoutput', 'satbias'), - 'source_file_fmt': '{source_dir}/{dump}.{year}{month}{day}/{hour}/atmos/{obs_type}_tlapmean.txt', - 'type': 'bc', - 'file_type': 'tlapse', - 'database': 'test', - 'provider': 'gsi', - 'experiment': 'gdasapp', - 'obs_types': [ - 'amsua_metop-b', - 'atms_npp', - 'cris-fsr_npp', - 'iasi_metop-c', - ], - } - tlapse_yaml = os.path.join(os.getcwd(), 'testinput', 'r2d2_store_tlapse.yaml') - with open(tlapse_yaml, 'w') as f: - yaml.dump(tlapse, f, sort_keys=False, default_flow_style=False) - # call the store commands - store_satbias(satbias_yaml) - store_satbias(tlapse_yaml) diff --git a/test/soca/CMakeLists.txt b/test/soca/CMakeLists.txt index 44f08cbea..74139a3ca 100644 --- a/test/soca/CMakeLists.txt +++ b/test/soca/CMakeLists.txt @@ -60,17 +60,6 @@ endforeach(FILENAME) install(FILES ${test_input} DESTINATION "test/testinput/") -# Create a test R2D2 database for obs -# ----------------------------------- -file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/obs) -set( OBS_DIR ${PROJECT_BINARY_DIR}/test/testdata ) -add_test(NAME test_gdasapp_soca_obsdb - COMMAND ${PROJECT_SOURCE_DIR}/test/soca/create_obsdb.py - WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/obs/) -set_tests_properties(test_gdasapp_soca_obsdb - PROPERTIES - ENVIRONMENT "PYTHONPATH=${PROJECT_BINARY_DIR}/ush:${PROJECT_SOURCE_DIR}/../../ush/python/wxflow/src:$ENV{PYTHONPATH};SOCA_TEST_OBS=${PROJECT_SOURCE_DIR}/test/soca/testdata") - # Test exgdas scripts from the global-worflow if (WORKFLOW_TESTS) add_subdirectory(gw) diff --git a/test/soca/create_obsdb.py b/test/soca/create_obsdb.py deleted file mode 100755 index 99f9179c4..000000000 --- a/test/soca/create_obsdb.py +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env python3 -import os -import shutil -from solo.configuration import Configuration -from solo.nice_dict import NiceDict -import ufsda.r2d2 -import subprocess - - -def cdl2nc(cdl_filename, nc4_filename): - try: - result = subprocess.run(["ncgen", "-o", nc4_filename, cdl_filename], capture_output=True) - print(result.stdout) - except subprocess.CalledProcessError as e: - print("Command failed:", e) - print("Return code:", e.returncode) - print("Output:", e.output) - - -if __name__ == "__main__": - - shared_root = './r2d2-shared' - r2d2_config_yaml = 'r2d2_config.yaml' - - # clean output before running - if nothing found, assume first run - try: - shutil.rmtree(shared_root) - os.remove(r2d2_config_yaml) - os.remove('*20180415.nc4') - except FileNotFoundError: - pass - - # Setup the shared R2D2 databases - ufsda.r2d2.setup(r2d2_config_yaml=r2d2_config_yaml, shared_root=shared_root) - - # Change the obs file name format - obsdir = os.getenv('SOCA_TEST_OBS') - cdl2nc(os.path.join(obsdir, 'adt.nc.cdl'), 'adt_j3_20180415.nc4') - cdl2nc(os.path.join(obsdir, 'adt.nc.cdl'), 'adt_j2_20180415.nc4') - cdl2nc(os.path.join(obsdir, 'sst.nc.cdl'), 'sst_noaa19_l3u_20180415.nc4') - cdl2nc(os.path.join(obsdir, 'sss.nc.cdl'), 'sss_smap_20180415.nc4') - cdl2nc(os.path.join(obsdir, 'prof.nc.cdl'), 'temp_profile_fnmoc_20180415.nc4') - cdl2nc(os.path.join(obsdir, 'prof.nc.cdl'), 'salt_profile_fnmoc_20180415.nc4') - cdl2nc(os.path.join(obsdir, 'icec.nc.cdl'), 'icec_EMC_20180415.nc4') - - # Create the test R2D2 database - obsstore = NiceDict({'start': '2018-04-15T00:00:00Z', - 'end': '2018-04-15T00:00:00Z', - 'step': 'PT24H', - 'source_dir': '.', - 'source_file_fmt': '{source_dir}/{obs_type}_{year}{month}{day}.nc4', - 'type': 'ob', - 'database': 'shared', - 'provider': 'gdasapp', - 'experiment': 'soca', - 'obs_types': ['adt_j3', - 'adt_j2', - 'sst_noaa19_l3u', - 'sss_smap', - 'temp_profile_fnmoc', - 'salt_profile_fnmoc', - 'icec_EMC', - 'icefb_GDR']}) - ufsda.r2d2.store(obsstore) - -# Create the test R2D2 database for output from bufr2ioda tests - obsstore['source_dir'] = '../../testoutput/' - obsstore['source_file_fmt'] = '{source_dir}/{obs_type}_{year}{month}{day}.nc' - obsstore['obs_types'] = ['temp_bufr_dbuoyprof', - 'salt_bufr_dbuoyprof', - 'temp_bufr_mbuoybprof', - 'salt_bufr_mbuoybprof', - 'bufr_sfcships', - 'bufr_sfcshipsu'] - ufsda.r2d2.store(obsstore) - - obsstore['start'] = '2018-04-01T00:00:00Z' - obsstore['end'] = '2018-04-01T00:00:00Z' - obsstore['source_file_fmt'] = '{source_dir}/{obs_type}_{year}{month}.nc' - obsstore['obs_types'] = ['bufr_tesacprof', - 'bufr_trkobprof'] - ufsda.r2d2.store(obsstore) diff --git a/test/soca/gw/CMakeLists.txt b/test/soca/gw/CMakeLists.txt index 68a5035a7..2d7d059d8 100644 --- a/test/soca/gw/CMakeLists.txt +++ b/test/soca/gw/CMakeLists.txt @@ -1,6 +1,5 @@ # test for creating an experiment directory within the global-workflow file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw/testrun) -file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw/concatioda) # Prepare COMROT and static dir add_test(NAME test_gdasapp_soca_prep @@ -21,14 +20,6 @@ IF (IS_DIRECTORY /scratch2/NCEPDEV/) set(PARTITION "hera") ENDIF() -# Test ioda-concatenation -add_test(NAME test_gdasapp_soca_concatioda - COMMAND ${Python3_EXECUTABLE} ${PROJECT_SOURCE_DIR}/test/soca/gw/test_concatioda.py ${PROJECT_BINARY_DIR}/test/soca/obs - WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw/concatioda) - set_tests_properties(test_gdasapp_soca_concatioda - PROPERTIES - ENVIRONMENT "PYTHONPATH=${PROJECT_BINARY_DIR}/ush:${PROJECT_BINARY_DIR}/lib/python${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}:$ENV{PYTHONPATH}") - # Clean-up add_test(NAME test_gdasapp_soca_run_clean COMMAND ${CMAKE_COMMAND} -E remove_directory ${PROJECT_BINARY_DIR}/test/soca/gw/testrun/testjjobs) @@ -45,8 +36,8 @@ add_test(NAME test_gdasapp_soca_setup_obsproc WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw/obsproc) # Test JGDAS_GLOBAL_OCEAN_ANALYSIS_* -set(jjob_list "JGDAS_GLOBAL_OCEAN_ANALYSIS_PREP" - "JGLOBAL_PREP_OCEAN_OBS" +set(jjob_list "JGLOBAL_PREP_OCEAN_OBS" + "JGDAS_GLOBAL_OCEAN_ANALYSIS_PREP" "JGDAS_GLOBAL_OCEAN_ANALYSIS_BMAT" "JGDAS_GLOBAL_OCEAN_ANALYSIS_RUN" "JGDAS_GLOBAL_OCEAN_ANALYSIS_CHKPT" diff --git a/test/soca/gw/run_jjobs.yaml.test b/test/soca/gw/run_jjobs.yaml.test index 683ad9135..ee42ad7f6 100644 --- a/test/soca/gw/run_jjobs.yaml.test +++ b/test/soca/gw/run_jjobs.yaml.test @@ -49,10 +49,7 @@ setup_expt config: SOCA_INPUT_FIX_DIR: @HOMEgfs@/sorc/gdas.cd/build/soca_static CASE_ANL: C48 SOCA_OBS_LIST: @HOMEgfs@/sorc/gdas.cd/parm/soca/obs/obs_list_small.yaml - COMIN_OBS: @HOMEgfs@/sorc/gdas.cd/build/test/soca/obs/r2d2-shared SOCA_NINNER: 1 - R2D2_OBS_SRC: gdasapp - R2D2_OBS_DUMP: soca SABER_BLOCKS_YAML: @HOMEgfs@/sorc/gdas.cd/parm/soca/berror/saber_blocks.yaml NICAS_RESOL: 1 NICAS_GRID_SIZE: 150 diff --git a/test/soca/gw/test_concatioda.py b/test/soca/gw/test_concatioda.py deleted file mode 100755 index 496ca82a1..000000000 --- a/test/soca/gw/test_concatioda.py +++ /dev/null @@ -1,29 +0,0 @@ -import ufsda -import ufsda.soca_utils -import os -import sys - - -# Define a function that duplicates a file -def duplicate_obs(src_filename): - if os.path.isfile(src_filename): - base_filename = os.path.basename(src_filename) - for nfile in list(range(2)): - print(nfile) - dest_filename = base_filename+'.'+str(nfile) - print(f"{src_filename} ---> {dest_filename}") - ufsda.disk_utils.copyfile(src_filename, dest_filename) - else: - raise Exception(f"{src_filename} does not exist") - - -# Get the directory where the files to be duplicated are located from command-line arguments -obsdir = sys.argv[1] - -# Test the concatenation for these obs types -obslist = ['adt_j3_20180415.nc4', 'sst_noaa19_l3u_20180415.nc4'] -for fname in obslist: - iodafname = os.path.join(obsdir, fname) - duplicate_obs(iodafname) - iodafname = os.path.join(obsdir, '..', 'gw', 'concatioda', fname) - ufsda.soca_utils.concatenate_ioda(iodafname) diff --git a/test/soca/testdata/adt.nc.cdl b/test/soca/testdata/adt.nc.cdl deleted file mode 100644 index 748e0a7a5..000000000 --- a/test/soca/testdata/adt.nc.cdl +++ /dev/null @@ -1,139 +0,0 @@ -netcdf adt.nc { -dimensions: - Location = 100 ; -variables: - int Location(Location) ; - Location:suggested_chunk_dim = 100LL ; - -// global attributes: - string :_ioda_layout = "ObsGroup" ; - :_ioda_layout_version = 0 ; - :nrecs = 1 ; - :odb_version = 1LL ; - :nlocs = 100LL ; - :nobs = 100LL ; - :history = "Fri Apr 19 08:49:14 2019: ncks -d nlocs,1,50000,500 adt.orig.nc -O adt.nc" ; - :NCO = "4.7.2" ; -data: - - Location = 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0 ; - - -group: MetaData { - variables: - int64 dateTime(Location) ; - dateTime:_FillValue = -3732782400LL ; - string dateTime:units = "seconds since 2018-04-15T12:00:00Z" ; - float latitude(Location) ; - latitude:_FillValue = 9.96921e+36f ; - float longitude(Location) ; - longitude:_FillValue = 9.96921e+36f ; - int sequenceNumber(Location) ; - sequenceNumber:_FillValue = -2147483647 ; - data: - - dateTime = 8979, 37065, -10967, -28959, -42772, -24069, 28606, -17781, - -32611, -31290, -13898, 8063, 17625, -23561, -39430, 22905, -18300, - -31232, -23225, -38636, -18325, 8982, -9841, 29332, -10620, 37498, - -1034, -23078, 14219, 14162, -21629, 36344, 23195, 31771, 10339, 17833, - 35703, -4052, -11549, 31202, 37231, 31403, 16181, 28642, 9593, -20577, - 8497, -3110, 17245, -39683, -24561, 8748, 23777, 5879, 17078, 600, - -21240, -21317, 41680, -32855, 29360, -41747, -12723, -18522, 7056, - -31791, -28776, -38385, -39168, -16351, 14451, 41413, 7784, -4273, - -36889, -17518, -30412, -30331, -22444, -43044, 7602, 371, 34001, - -10269, -36806, -43073, 8321, -11562, -11838, 42641, -30902, 16071, - 16374, -22526, -33032, -28369, -17501, -16309, 10748, 42986 ; - - latitude = -42.30449, -61.53271, -54.52587, 2.40806, -13.3181, -65.35713, - -13.49405, -51.82866, 11.22875, -51.14191, 65.89512, 1.492048, - -28.5202, -58.87149, 14.84761, -59.58656, -28.63931, -53.45946, - -45.69531, -23.91795, -27.42865, -42.44091, -50.2743, -47.36231, - -64.71016, -45.25023, 46.92305, -39.15643, 30.07972, 32.76525, - 30.69972, -57.81081, -65.82172, 3.266878, -52.58415, -18.54302, - -30.36827, -60.3972, -28.85406, -24.43554, -56.09284, -14.73257, - -60.25863, -15.23002, -64.27747, 66.14738, -19.70168, -50.90127, - -45.91634, 27.03309, -50.4441, -31.67368, -54.68562, 53.79114, - -52.86835, 36.03845, 48.28653, 44.95198, 6.986401, 23.08882, -48.57338, - 36.24638, 28.12075, -17.97434, 48.89173, -28.65383, 11.38542, - -35.75763, 1.995264, -40.03509, 18.95174, 19.98642, 15.14227, - -52.50491, -52.90968, -61.13697, -62.06583, -59.66623, -8.916806, - -26.4584, 23.91177, 46.33038, 50.50362, -64.24638, -49.51569, -27.8661, - -11.13248, -28.27375, -14.97766, -39.31322, -63.87175, -56.55973, - -64.89208, -12.89731, 31.52845, 30.96179, -61.62994, -38.08575, - -34.69767, -53.99775 ; - - longitude = -3.373801, -19.74908, 95.06168, -46.61552, 4.35299, -173.69, - -103.4871, 119.5747, 142.6289, 175.3936, -29.76636, -23.91055, - 102.6672, -117.4293, 169.591, -36.75309, 101.5004, 178.4957, -98.71407, - -175.7098, 100.9004, -3.265874, -160.1637, -83.9273, 124.6124, - 6.571999, -136.8072, -93.49664, -63.95547, -65.41, -63.2631, -97.47237, - -6.842985, 58.5721, 111.8786, 107.1669, -124.3838, 78.7492, 73.26094, - 47.92515, -6.949519, 51.99478, -6.666136, -102.8136, 36.91441, - 6.947506, -15.96541, 170.7399, 91.00462, 164.346, 146.2024, -10.29815, - 52.1002, -156.9272, 83.14131, -10.65498, -50.38128, -53.59933, - -167.6185, 137.828, -82.67605, 24.96964, 50.42411, 96.70792, -49.4618, - 158.2, -43.34373, -169.5071, 174.3347, -122.4666, -58.82631, -172.6444, - -28.94908, 63.77332, -50.14354, 137.584, -98.06361, -90.98943, - -79.03435, -1.207335, -32.62028, -18.51745, -164.6857, 167.3599, - -45.90093, -1.890546, -19.34283, 72.9696, 67.16569, -147.3289, - -155.184, -14.87442, 12.47116, -80.52476, 133.6994, -34.77795, - 139.1339, -121.1142, 127.6422, -132.5225 ; - - } // group MetaData - -group: ObsError { - variables: - float absoluteDynamicTopography(Location) ; - absoluteDynamicTopography:_FillValue = 9.96921e+36f ; - string absoluteDynamicTopography:units = "m" ; - data: - - absoluteDynamicTopography = 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, - 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, - 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, - 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, - 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, - 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, - 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, - 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1 ; - } // group ObsError - -group: ObsValue { - variables: - float absoluteDynamicTopography(Location) ; - absoluteDynamicTopography:_FillValue = 9.96921e+36f ; - string absoluteDynamicTopography:units = "m" ; - data: - - absoluteDynamicTopography = 0.2886, -1.3556, -0.5692, 0.5597, 0.3317, - -0.9281, 0.646, -0.3841, 1.1805, 0.2265, -0.3182, 0.5563, 1.1411, - -0.4136, 1.2235, -1.1775, 0.8149, -0.0451, 0.4549, 1.1222, 0.5479, - 0.2928, 0.6877, 0.2859, -1.2493, -0.1018, 0.4664, 0.7455, 0.7465, - 0.5585, 0.6794, -0.0422, -1.4293, 0.8899, -0.9427, 1.1056, 0.9883, - -1.0632, 1.0919, 1.3852, -0.8461, 0.9989, -1.4867, 0.7519, -1.2997, - -0.1713, 0.5795, 0.5559, 0.3537, 1.2151, 0.2197, 0.8687, -0.7904, - 0.1346, -0.9257, 0.1562, -0.1302, -0.0695, 0.9187, 1.1942, 0.3083, - 0.2067, 0.5803, 0.8152, -0.4867, 1.291, 0.5355, 0.6636, 1.2767, 0.6614, - 0.6591, 1.3125, 0.3322, -0.6027, -0.1315, -0.9311, -0.519, -0.507, - 0.5987, 0.6005, 0.4232, 0.1729, 0.2895, -1.1387, -0.3301, 0.5248, - 0.4213, 1.0189, 0.9698, 0.6685, -1.1554, -0.942, -1.3066, 0.6466, - 1.469, 0.2632, -0.8869, 0.6544, 0.6571, -0.1728 ; - } // group ObsValue - -group: PreQC { - variables: - int absoluteDynamicTopography(Location) ; - absoluteDynamicTopography:_FillValue = -2147483647 ; - data: - - absoluteDynamicTopography = 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ; - } // group PreQC -} diff --git a/test/soca/testdata/icec.nc.cdl b/test/soca/testdata/icec.nc.cdl deleted file mode 100644 index b89e4ea79..000000000 --- a/test/soca/testdata/icec.nc.cdl +++ /dev/null @@ -1,139 +0,0 @@ -netcdf icec.nc { -dimensions: - Location = 100 ; - nvars = 1 ; -variables: - int Location(Location) ; - Location:suggested_chunk_dim = 100LL ; - float nvars(nvars) ; - nvars:suggested_chunk_dim = 100LL ; - -// global attributes: - string :_ioda_layout = "ObsGroup" ; - :_ioda_layout_version = 0 ; - :converter = "somthing.py" ; - :nvars = 1 ; - :date_time = 2018041512 ; - :nlocs = 100 ; - :nobs = 100 ; - :history = "Wed May 12 11:02:49 2021: ncks -O -4 icec.nc icec.nc4" ; - :NCO = "netCDF Operators version 4.7.9 (Homepage = http://nco.sf.net, Code = http://github.com/nco/nco)" ; -data: - - Location = 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0 ; - - nvars = 0 ; - -group: MetaData { - variables: - int64 dateTime(Location) ; - dateTime:_FillValue = -2208988800LL ; - string dateTime:units = "seconds since 1970-01-01T00:00:00Z" ; - float latitude(Location) ; - latitude:_FillValue = 9.96921e+36f ; - float longitude(Location) ; - longitude:_FillValue = 9.96921e+36f ; - string variable_names(nvars) ; - string variable_names:_FillValue = "" ; - data: - - dateTime = 1523756403, 1523755138, 1523809850, 1523761241, 1523822234, - 1523768790, 1523785038, 1523759437, 1523766482, 1523810666, 1523796805, - 1523793968, 1523823489, 1523752584, 1523775969, 1523762155, 1523768587, - 1523814065, 1523804452, 1523785153, 1523814803, 1523798982, 1523809343, - 1523817342, 1523781349, 1523820098, 1523791041, 1523769787, 1523815706, - 1523778971, 1523804767, 1523796237, 1523824707, 1523818565, 1523751405, - 1523787456, 1523810909, 1523762957, 1523791035, 1523792510, 1523786864, - 1523790617, 1523814394, 1523791642, 1523755808, 1523810550, 1523762563, - 1523777575, 1523781304, 1523777512, 1523783413, 1523800295, 1523767606, - 1523780294, 1523787500, 1523805171, 1523833527, 1523761005, 1523774017, - 1523834116, 1523751669, 1523750758, 1523755978, 1523756554, 1523801484, - 1523818162, 1523824621, 1523818222, 1523806101, 1523766245, 1523832461, - 1523793475, 1523804854, 1523775300, 1523766009, 1523777548, 1523817482, - 1523779036, 1523830924, 1523807451, 1523821977, 1523793607, 1523832064, - 1523814637, 1523802742, 1523807362, 1523807507, 1523802980, 1523780526, - 1523795959, 1523758548, 1523761287, 1523830764, 1523831286, 1523762363, - 1523833304, 1523750667, 1523809587, 1523760587, 1523798073 ; - - latitude = -78.375, -77.375, -76.625, -76.125, -75.625, -75.125, -74.875, - -74.375, -74.125, -73.625, -73.375, -72.875, -72.625, -72.375, -72.125, - -71.875, -71.625, -71.375, -70.875, -70.625, -70.375, -70.125, -69.875, - -69.625, -69.375, -69.125, -68.625, -68.375, -68.125, -67.625, -66.875, - -66.375, -66.125, -65.625, -65.375, -64.875, -64.375, -63.125, 49.125, - 52.625, 55.125, 57.125, 58.875, 60.375, 61.875, 63.125, 64.375, 65.625, - 67.125, 67.875, 68.625, 69.375, 69.875, 70.375, 70.625, 71.125, 71.625, - 72.125, 72.375, 72.875, 73.375, 73.875, 74.375, 74.875, 75.375, 75.875, - 76.125, 76.375, 76.875, 77.125, 77.375, 77.625, 77.875, 78.125, 78.625, - 78.875, 79.125, 79.375, 79.625, 79.875, 80.375, 80.625, 80.875, 81.125, - 81.375, 81.625, 81.875, 82.125, 82.375, 82.875, 83.125, 83.375, 83.875, - 84.125, 84.625, 84.875, 85.375, 85.625, 85.875, 86.125 ; - - longitude = 178.625, 185.875, 163.125, 184.625, 193.125, 184.375, 334.375, - 224.125, 339.375, 221.875, 315.125, 180.125, 217.375, 242.625, 269.625, - 284.125, 322.625, 337.375, 179.125, 195.625, 231.375, 243.125, 256.375, - 285.875, 314.875, 353.125, 4.375, 73.125, 311.875, 315.375, 68.375, - 76.625, 300.625, 83.875, 142.625, 92.375, 116.875, 309.875, 306.625, - 145.875, 147.625, 143.625, 158.875, 159.375, 276.875, 297.125, 195.375, - 181.375, 73.375, 336.625, 295.375, 56.625, 162.375, 55.625, 299.625, - 238.125, 190.625, 49.625, 284.875, 339.625, 344.625, 263.375, 128.375, - 21.625, 75.875, 32.625, 150.125, 277.625, 14.375, 34.375, 58.375, - 93.875, 165.875, 254.125, 23.875, 46.125, 63.125, 101.125, 160.625, - 294.875, 38.625, 80.875, 119.625, 153.125, 156.875, 182.375, 197.375, - 271.375, 346.625, 24.125, 151.125, 318.875, 107.125, 176.375, 131.875, - 187.375, 3.375, 144.875, 164.625, 251.625 ; - - variable_names = "sea_ice_area_fraction" ; - } // group MetaData - -group: ObsError { - variables: - float seaIceFraction(Location) ; - seaIceFraction:_FillValue = 9.96921e+36f ; - data: - - seaIceFraction = 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, - 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, - 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, - 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, - 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, - 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, - 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, - 0.1, 0.1, 0.1, 0.1, 0.1 ; - } // group ObsError - -group: ObsValue { - variables: - float seaIceFraction(Location) ; - seaIceFraction:_FillValue = 9.96921e+36f ; - data: - - seaIceFraction = 0.66, 0.86, 0.72, 0.94, 0.92, 0.94, 0.7, 0.72, 0.65, - 0.81, 0.89, 0.87, 0.59, 0.89, 0.81, 0.51, 0.88, 0.32, 0.76, 0.78, 0.4, - 0.4, 0.74, 0.52, 0.81, 0.48, 0.35, 0.66, 0.83, 0.72, 0.62, 0.78, 0.86, - 0.8, 0.35, 0.64, 0.25, 0.39, 0.28, 0.72, 0.25, 0.96, 0.21, 0.26, - 0.9899999, 0.98, 0.67, 0.39, 0.97, 0.33, 0.98, 0.77, 0.9899999, 0.66, - 0.9899999, 0.9899999, 0.9899999, 0.22, 0.74, 0.92, 0.65, 0.9899999, - 0.9899999, 0.38, 0.9899999, 0.22, 0.98, 0.98, 0.3, 0.8, 0.85, 0.98, - 0.96, 0.9899999, 0.84, 0.93, 0.9899999, 0.84, 0.98, 0.8, 0.95, - 0.9899999, 0.9899999, 0.9899999, 0.9899999, 0.98, 0.9899999, 0.94, - 0.9899999, 0.9899999, 0.9899999, 0.9899999, 0.9899999, 0.9899999, - 0.9899999, 0.9899999, 0.98, 0.9899999, 0.9899999, 0.9899999 ; - } // group ObsValue - -group: PreQC { - variables: - float seaIceFraction(Location) ; - seaIceFraction:_FillValue = 9.96921e+36f ; - data: - - seaIceFraction = 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0 ; - } // group PreQC -} diff --git a/test/soca/testdata/prof.nc.cdl b/test/soca/testdata/prof.nc.cdl deleted file mode 100644 index 2032b235a..000000000 --- a/test/soca/testdata/prof.nc.cdl +++ /dev/null @@ -1,388 +0,0 @@ -netcdf prof.nc { -dimensions: - Location = 218 ; - nvars = 2 ; -variables: - int Location(Location) ; - Location:suggested_chunk_dim = 218LL ; - float nvars(nvars) ; - nvars:suggested_chunk_dim = 100LL ; - -// global attributes: - string :_ioda_layout = "ObsGroup" ; - :_ioda_layout_version = 0 ; - :nrecs = 20 ; - :nvars = 2 ; - :odb_version = 1LL ; - :date_time = 2018041512 ; - :nlocs = 218 ; - :nobs = 436 ; -data: - - Location = 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0 ; - - nvars = 0, 0 ; - -group: MetaData { - variables: - int64 dateTime(Location) ; - dateTime:_FillValue = -3732782400LL ; - string dateTime:units = "seconds since 2018-04-15T12:00:00Z" ; - string date_time(Location) ; - string date_time:_FillValue = "" ; - float depth(Location) ; - depth:_FillValue = 9.96921e+36f ; - float latitude(Location) ; - latitude:_FillValue = 9.96921e+36f ; - float longitude(Location) ; - longitude:_FillValue = 9.96921e+36f ; - int sequenceNumber(Location) ; - sequenceNumber:_FillValue = -2147483647 ; - string variable_names(nvars) ; - string variable_names:_FillValue = "" ; - data: - - dateTime = -28800, -28800, -28800, -28800, -28800, -28800, -28800, -28800, - -28800, -28800, -28800, -32400, -32400, -32400, -32400, -32400, -32400, - -32400, -32400, -32400, -32400, -21600, -21600, -21600, -21600, -21600, - -21600, -21600, -21600, -21600, -21600, -21600, -21600, -14400, -14400, - -14400, -14400, -14400, -14400, -14400, -14400, -14400, -14400, 7200, - 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 14400, - 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 10800, 10800, - 10800, 10800, 10800, 10800, 10800, 10800, 10800, 3600, 3600, 3600, - 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 18000, 18000, - 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, - 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 3600, - 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 7200, 7200, 7200, - 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 28800, - 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, - 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 25200, - 25200, 25200, 25200, 25200, 25200, 25200, 25200, 25200, 25200, 32400, - 32400, 32400, 32400, 32400, 32400, 32400, 32400, 32400, 32400, 32400, - 39600, 39600, 39600, 39600, 39600, 39600, 39600, 39600, 39600, 39600, - 39600, 39600, 39600, 39600, 39600, 39600, 39600, 39600, 39600, 39600, - 39600, 39600, 39600, 36000, 36000, 36000, 36000, 36000, 36000, 36000, - 36000, 36000, 36000, 36000, 36000, 36000, 36000, 36000, 36000, 36000, - 36000, 36000, 36000, 36000, 36000, 36000, 36000, 36000, 36000 ; - - depth = 60, 500, 40, 100, 180, 80, 1, 140, 120, 300, 20, 125, 250, 25, - 150, 200, 1, 300, 50, 500, 75, 10, 300, 140, 120, 500, 100, 80, 180, - 60, 40, 20, 1, 100, 120, 140, 180, 20, 500, 40, 60, 300, 80, 40, 500, - 60, 180, 20, 120, 140, 1, 80, 300, 100, 25, 100, 50, 1, 300, 250, 500, - 125, 200, 75, 25, 500, 250, 150, 300, 50, 200, 100, 100, 180, 300, 40, - 60, 10, 20, 1, 120, 140, 500, 80, 500, 150, 300, 250, 125, 75, 100, 50, - 200, 25, 100, 50, 125, 75, 300, 250, 500, 25, 200, 150, 1, 100, 200, - 50, 1.5, 125, 75, 300, 25, 150, 250, 140, 80, 500, 300, 1, 100, 40, 60, - 10, 20, 180, 5, 120, 300, 140, 1, 120, 100, 80, 500, 40, 180, 60, 80, - 100, 300, 120, 500, 140, 20, 1, 180, 40, 100, 50, 200, 1.5, 125, 75, - 150, 300, 25, 250, 500, 40, 100, 80, 140, 120, 1, 20, 300, 180, 60, - 100, 80, 1, 140, 300, 120, 20, 500, 60, 180, 300, 100, 120, 500, 25, - 140, 20, 5, 40, 180, 60, 45, 80, 60, 300, 80, 4, 1, 120, 5, 140, 100, - 10, 20, 180, 40, 500, 60, 300, 80, 100, 1, 120, 140, 180, 500, 10, 20, 40 ; - - latitude = 8.1, 8.1, 8.1, 8.1, 8.1, 8.1, 8.1, 8.1, 8.1, 8.1, 8.1, -8, -8, - -8, -8, -8, -8, -8, -8, -8, -8, -4.03686, -4.03686, -4.03686, -4.03686, - -4.03686, -4.03686, -4.03686, -4.03686, -4.03686, -4.03686, -4.03686, - -4.03686, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, -5, -5, -5, -5, -5, -5, -5, -5, - -5, -5, -5, 2, 2, 2, 2, 2, 2, 2, 2, 2, -5, -5, -5, -5, -5, -5, -5, -5, - -5, -8.01557, -8.01557, -8.01557, -8.01557, -8.01557, -8.01557, - -8.01557, -8.01557, -8.01557, -8.01557, -8.01557, -8.01557, -2, -2, -2, - -2, -2, -2, -2, -2, -2, -2, -2.2, -2.2, -2.2, -2.2, -2.2, -2.2, -2.2, - -2.2, -2.2, -2.2, -2.2, -0.0141, -0.0141, -0.0141, -0.0141, -0.0141, - -0.0141, -0.0141, -0.0141, -0.0141, -0.0141, 0.00453, 0.00453, 0.00453, - 0.00453, 0.00453, 0.00453, 0.00453, 0.00453, 0.00453, 0.00453, 0.00453, - 0.00453, 0.00453, 2, 2, 2, 2, 2, 2, 2, 2, 2, -8, -8, -8, -8, -8, -8, - -8, -8, -8, -8, -8, -0.01599, -0.01599, -0.01599, -0.01599, -0.01599, - -0.01599, -0.01599, -0.01599, -0.01599, -0.01599, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, -6.0014, -6.0014, -6.0014, -6.0014, -6.0014, -6.0014, -6.0014, - -6.0014, -6.0014, -6.0014, -6.0014, -6.0014, -6.0014, -6.0014, - -4.03584, -4.03584, -4.03584, -4.03584, -4.03584, -4.03584, -4.03584, - -4.03584, -4.03584, -4.03584, -4.03584, -4.03584 ; - - longitude = -125, -125, -125, -125, -125, -125, -125, -125, -125, -125, - -125, 164.8, 164.8, 164.8, 164.8, 164.8, 164.8, 164.8, 164.8, 164.8, - 164.8, 67.24096, 67.24096, 67.24096, 67.24096, 67.24096, 67.24096, - 67.24096, 67.24096, 67.24096, 67.24096, 67.24096, 67.24096, -140, -140, - -140, -140, -140, -140, -140, -140, -140, -140, -139.9, -139.9, -139.9, - -139.9, -139.9, -139.9, -139.9, -139.9, -139.9, -139.9, -139.9, 165.2, - 165.2, 165.2, 165.2, 165.2, 165.2, 165.2, 165.2, 165.2, 165.2, 165.2, - 165.2, 165.2, 165.2, 165.2, 165.2, 165.2, 165.2, 80.45833, 80.45833, - 80.45833, 80.45833, 80.45833, 80.45833, 80.45833, 80.45833, 80.45833, - 80.45833, 80.45833, 80.45833, -155, -155, -155, -155, -155, -155, -155, - -155, -155, -155, -170, -170, -170, -170, -170, -170, -170, -170, -170, - -170, -170, 155.9603, 155.9603, 155.9603, 155.9603, 155.9603, 155.9603, - 155.9603, 155.9603, 155.9603, 155.9603, -22.98389, -22.98389, - -22.98389, -22.98389, -22.98389, -22.98389, -22.98389, -22.98389, - -22.98389, -22.98389, -22.98389, -22.98389, -22.98389, -110, -110, - -110, -110, -110, -110, -110, -110, -110, -110.1, -110.1, -110.1, - -110.1, -110.1, -110.1, -110.1, -110.1, -110.1, -110.1, -110.1, - 155.957, 155.957, 155.957, 155.957, 155.957, 155.957, 155.957, 155.957, - 155.957, 155.957, -140, -140, -140, -140, -140, -140, -140, -140, -140, - -140, -140, -125.1, -125.1, -125.1, -125.1, -125.1, -125.1, -125.1, - -125.1, -125.1, -125.1, -139.9, -139.9, -139.9, -139.9, -139.9, -139.9, - -139.9, -139.9, -139.9, -139.9, -139.9, -139.9, -139.9, 8.00453, - 8.00453, 8.00453, 8.00453, 8.00453, 8.00453, 8.00453, 8.00453, 8.00453, - 8.00453, 8.00453, 8.00453, 8.00453, 8.00453, 67.242, 67.242, 67.242, - 67.242, 67.242, 67.242, 67.242, 67.242, 67.242, 67.242, 67.242, 67.242 ; - - sequenceNumber = 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, - 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 11, 11, 11, 11, - 11, 11, 11, 11, 11, 11, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, - 12, 13, 13, 13, 13, 13, 13, 13, 13, 13, 14, 14, 14, 14, 14, 14, 14, 14, - 14, 14, 14, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 16, 16, 16, 16, 16, - 16, 16, 16, 16, 16, 16, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 18, 18, - 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 19, 19, 19, 19, 19, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20 ; - - variable_names = "sea_water_salinity", "sea_water_temperature" ; - } // group MetaData - -group: ObsError { - variables: - float salinity(Location) ; - salinity:_FillValue = 9.96921e+36f ; - float waterTemperature(Location) ; - waterTemperature:_FillValue = 9.96921e+36f ; - data: - - salinity = 0.2522745, 0.1063194, 0.2849693, 0.166195, 0.09365788, - 0.200834, 0.2371611, 0.08366414, 0.1299536, 0.1010965, 0.1687779, - 0.7850429, 0.4356464, 1.591666, 0.8089812, 0.6832849, 1.697105, - 0.3133486, 1.511612, 0.2497392, 1.203681, 0.2197831, 0.1222905, - 0.08824342, 0.1645312, 0.1172962, 0.1364866, 0.1435371, 0.1202509, - 0.2682254, 0.1998587, 0.1259456, 0.6451571, 1.210507, 1.097976, - 0.8719119, 0.4586407, 1.158771, 0.1140179, 1.141717, 1.164443, - 0.1429435, 1.197306, 0.2177143, 0.1228545, 0.2259728, 0.1939958, - 0.2478939, 0.2443465, 0.2529505, 0.3637651, 0.2088105, 0.1383288, - 0.1579156, 0.1400251, 0.27416, 0.2761041, 0.2834496, 0.1044707, - 0.1786518, 0.117582, 0.1529245, 0.224429, 1.179957, 1.548487, - 0.2490613, 0.510761, 0.9821573, 0.3357151, 1.441269, 0.880218, 0.92051, - 0.1224098, 0.0733786, 0.09151919, 0.3130135, 0.3720615, 0.1560941, - 0.1031811, 0.2011424, 0.2196218, 0.06576524, 0.09795111, 0.1263566, - 0.1404088, 1.438963, 0.1937371, 0.2689984, 1.490757, 1.047074, - 1.288639, 1.00524, 0.6400538, 1.021853, 0.1832772, 0.181006, 0.1896285, - 0.2019524, 0.1270845, 0.2043958, 0.1171445, 0.2605884, 0.2675938, - 0.2263254, 0.3653024, 0.2850498, 0.2288099, 0.09864562, 0.1805917, - 0.3270485, 0.1413604, 0.1385984, 0.215419, 0.1264153, 0.1776507, - 0.2440303, 0.391326, 0.174588, 0.1708031, 0.05932364, 0.3705409, - 0.2244651, 0.1368625, 0.1493623, 0.05727173, 0.166375, 0.05866817, - 0.2646765, 0.1636383, 0.6282568, 1.789018, 0.735101, 0.8842987, - 1.167529, 0.1027457, 1.507743, 0.4915881, 1.568499, 1.363651, 1.080779, - 0.1952056, 0.9173244, 0.06207152, 0.757747, 1.901986, 1.914773, - 0.5039916, 1.71778, 0.2737243, 0.09864481, 0.2359564, 0.3332464, - 0.3405225, 0.1123013, 0.1469922, 0.0873469, 0.2929741, 0.1750725, - 0.1346339, 0.138216, 0.2096565, 0.1308088, 0.1126329, 0.2334442, - 0.2602245, 0.1921947, 0.1253058, 0.09266749, 0.155894, 0.1143336, - 0.1697396, 0.2593261, 0.1232226, 0.1157582, 0.1466153, 0.1364038, - 0.1281689, 0.2194774, 0.09342678, 0.1313902, 0.05775158, 0.09673796, - 0.1344558, 0.3084517, 0.2484634, 0.3372627, 0.3816409, 0.1399285, - 0.1459216, 0.2595628, 0.151517, 0.2104556, 0.1805469, 0.1823283, - 0.2343753, 1.871043, 1.056106, 0.1652864, 1.759668, 0.2059344, - 0.1846005, 0.8090061, 0.5420793, 0.1961852, 0.1148487, 0.1676946, - 0.1995637, 0.1171088, 0.2132636, 0.1414041, 0.6741951, 0.2209578, - 0.06775443, 0.1180433, 0.1100427, 0.2316091, 0.2603508, 0.1870268 ; - - waterTemperature = 1.743426, 0.5193261, 1.494207, 0.8796794, 0.5286709, - 1.343593, 0.5824349, 0.61001, 0.667032, 0.5093725, 0.8844231, - 0.9340492, 1.007331, 0.468168, 0.9813886, 1.036275, 0.5447601, - 0.770175, 0.3583136, 0.6713341, 0.8158218, 0.7916454, 0.6055509, - 0.7106637, 0.7956954, 0.5452055, 1.114186, 1.488101, 0.6829131, - 1.468622, 1.12228, 0.7781153, 0.9701674, 1.599963, 1.476451, 0.7450125, - 0.5824413, 0.644627, 0.6039046, 0.8115129, 0.903461, 0.5930381, - 1.065919, 0.2801136, 0.5537068, 0.737125, 0.8197537, 0.3324322, - 1.362025, 1.135762, 0.3594703, 1.020024, 0.6098969, 1.323719, - 0.4565539, 0.5973899, 0.5753483, 0.272785, 0.5647165, 0.9110706, - 0.5528083, 1.167597, 1.235241, 0.5491017, 0.4130876, 0.589695, - 1.195557, 0.9150438, 0.7125937, 0.4867784, 1.229869, 0.6954335, - 0.9935142, 0.6311837, 0.5309837, 1.439539, 1.658565, 0.6073291, - 0.5337266, 0.7063531, 0.8778601, 0.7684838, 0.4986047, 1.235635, - 0.5999914, 1.308395, 0.5793702, 0.7610333, 0.9590447, 0.6941347, - 0.7923799, 0.4928668, 1.169785, 0.2800499, 0.7219963, 0.3700807, - 0.8848718, 0.4830801, 0.5760365, 0.8778846, 0.5401599, 0.3846734, - 1.235485, 1.295008, 0.3513189, 0.466359, 1.139407, 0.4031729, - 0.5074973, 1.204991, 0.4061031, 0.6839842, 0.4782485, 1.380711, - 0.8387715, 0.7210994, 1.324711, 0.6442875, 0.6358635, 1.226067, - 1.129524, 1.05598, 1.353591, 0.5635409, 0.6810182, 0.6161842, - 0.9409899, 0.9163142, 0.5753275, 0.5561668, 1.119911, 0.5457644, - 0.6782846, 1.336049, 0.5781834, 1.43989, 0.5525411, 1.019478, 1.248575, - 1.2796, 0.5426413, 1.302236, 0.5345117, 1.036997, 0.6088989, 0.4321637, - 0.6309587, 0.7555988, 0.4618859, 0.403173, 1.153354, 0.3227628, - 1.173669, 0.4006487, 1.37786, 0.6794108, 0.3855352, 0.8370423, - 0.6134599, 0.7724864, 1.520247, 1.046432, 0.8824214, 1.474335, - 0.5206715, 0.4941791, 0.5990528, 0.5935799, 0.8832166, 1.424629, - 1.542141, 0.8646343, 0.6747602, 0.5828347, 0.8573534, 0.8243105, - 0.5863172, 1.030508, 0.5895203, 0.6056085, 1.342003, 1.028111, - 0.6016723, 0.527028, 1.005599, 0.5325223, 0.5584281, 0.5712408, - 0.7219398, 0.8380678, 0.6462824, 1.317487, 0.8282857, 0.6612134, - 0.7838165, 1.337059, 1.043711, 0.6533467, 1.208123, 0.638188, - 0.6995534, 1.724898, 1.705356, 0.6825911, 1.242171, 0.6282559, 1.50075, - 0.5917126, 1.514512, 1.116175, 0.5734112, 0.8052191, 0.656361, - 0.668393, 0.5316752, 0.5864132, 0.8440492, 1.121295 ; - } // group ObsError - -group: ObsValue { - variables: - float salinity(Location) ; - salinity:_FillValue = 9.96921e+36f ; - float waterTemperature(Location) ; - waterTemperature:_FillValue = 9.96921e+36f ; - data: - - salinity = 34.57398, 34.60444, 34.41549, 34.71024, 34.73309, 34.662, - 34.34, 34.74724, 34.74189, 34.7036, 34.24408, 35.5955, 35.39627, - 34.81105, 35.6922, 35.70549, 34.74729, 35.04684, 34.93018, 34.64479, - 35.09085, 35.47, 35.0326, 35.15076, 35.16211, 34.8932, 35.23, 35.20581, - 35.15009, 35.18, 35.49, 35.05608, 34.98, 35.07314, 34.85984, 34.85921, - 34.88235, 34.99768, 34.61454, 35.01868, 35.04686, 34.82505, 35.07153, - 35.2707, 34.63697, 35.37906, 35.02031, 35.21252, 35.3908, 35.29946, - 35.5, 35.45095, 34.8038, 35.52097, 34.64267, 35.20449, 34.78244, 34.83, - 34.80511, 34.77232, 34.66197, 35.34407, 35.05775, 35.1338, 34.75198, - 34.61681, 35.2095, 35.6226, 34.84489, 34.90092, 35.87683, 35.35735, - 34.69, 34.86082, 34.85189, 34.26, 34.66, 34.25, 34.25, 34.22, 34.85972, - 34.86072, 34.7815, 34.88726, 34.59576, 35.50129, 34.79575, 34.79715, - 35.4471, 35.26348, 35.33844, 35.21389, 35.02264, 35.18464, 35.43448, - 35.26216, 35.48049, 35.34953, 34.76404, 34.85212, 34.62323, 35.22366, - 35.13306, 35.57404, 35.59, 34.81, 35.11, 34.76, 34.58, 35.05, 34.64, - 34.87, 34.63, 35.4, 34.93, 35.50653, 36.38, 34.69085, 35.17369, 36.34, - 35.78912, 36.34, 36.5, 36.34, 36.32, 35.40076, 36.34, 35.75, 34.83435, - 34.89127, 33.98383, 34.91024, 34.93637, 34.95012, 34.65226, 34.64199, - 34.87164, 35.51104, 35.61652, 35.54525, 34.79385, 35.31134, 34.66566, - 34.98342, 35.08948, 34.9439, 34.86064, 35.28613, 34.75, 34.77, 35.11, - 34.36, 35.01, 34.64, 35.4, 34.88, 34.63, 34.89, 34.61314, 35.01323, - 35.08269, 35.07761, 34.87947, 34.9267, 35.13, 35.0029, 34.83858, - 34.89431, 35.04663, 34.93151, 34.92387, 34.79, 34.87672, 34.83274, - 34.90145, 34.66761, 34.63993, 34.84803, 34.85432, 34.86982, 35.09964, - 35.33, 34.64524, 35.07434, 35.08952, 35.07367, 35.33, 35.28, 34.97347, - 35.25, 35.0976, 35.33, 35.93, 35.11495, 35.88, 32.29442, 32.71, 35.77, - 34.71, 35.64164, 35.72212, 34.94, 35.95, 35.54502, 35.98, 34.68404, - 35.16, 35.02942, 35.19083, 35.32, 34.97, 35.14178, 35.14002, 35.13965, - 34.91897, 35.51, 35.05788, 35.32 ; - - waterTemperature = 18.94, 8.149994, 25.89001, 12.94, 11.30002, 14.42001, - 28.22, 12.01001, 12.39001, 10.25, 27.92999, 26.25, 17.27002, 29.42001, - 25.02002, 21.20999, 29.70999, 13.76001, 29.39999, 8.540009, 29.45999, - 29.97, 11.80002, 15.84, 16.70999, 9.220001, 17.36002, 20.63, 15.09, - 25.67999, 28.75, 29.85001, 30.67001, 21.70999, 14.85001, 13.86002, - 13.03, 26.5, 7.730011, 26.08002, 24.77002, 11.39999, 23.86002, - 27.74002, 8.110016, 27.74002, 14.10001, 27.74002, 20.34, 17.99002, - 27.72, 26.47, 10.87, 24.57001, 29.87, 28.78, 29.60001, 29.87, 11.45999, - 12.24002, 8.779999, 28.67001, 17.91, 29.57001, 29.92001, 8.070007, - 15.52002, 27.69, 11.47, 29.84, 24.01001, 29.29001, 15.58002, 12.26001, - 10.45001, 28.35001, 20.79001, 28.64999, 28.58002, 28.92001, 14.34, - 13.54001, 8.660004, 17.22, 7.52002, 23.42999, 11.08002, 11.32001, - 25.39001, 27.10001, 26.20001, 27.45001, 14.85001, 27.45001, 28.13, - 28.47, 26.83002, 28.33002, 10.67999, 12.14001, 8.300018, 28.39999, - 16.5, 25.70999, 28.38, 29.67999, 16.94, 29.79001, 30.12, 29.62, 29.78, - 12.27002, 29.91, 23.85001, 13.63, 15.27002, 20.57001, 7.630005, - 12.20001, 28.36002, 17.80002, 26.94, 24.04001, 27.72, 27.58002, 14.34, - 27.78, 16.24002, 11.86002, 13.97, 28.62, 14.23001, 14.45001, 15.23001, - 8.670013, 24.42999, 13.44, 25.57001, 23.22, 20.17001, 10.92001, 17.47, - 8.589996, 14.11002, 26.91, 26.99002, 12.51001, 26.29001, 29.70001, - 29.80002, 17.02002, 29.89999, 29.63, 29.78, 24.29001, 12.38, 29.91, - 13.70999, 7.700012, 26.26001, 22.04001, 24.19, 14.22, 16.35001, - 26.54001, 26.41, 11.58002, 13.20001, 24.92999, 16.39999, 22.87, - 27.77002, 14.32001, 11.64999, 15, 26.77002, 8.360016, 25.16, 13.47, - 11.99002, 19.89999, 18.64001, 8.330017, 26.28, 16.95999, 26.29001, - 26.45999, 26.12, 14.08002, 25.75, 26.05002, 24.57001, 18.53, 11.66, - 17.61002, 29.07001, 29.34, 16.47, 28.64999, 16.02002, 16.95001, 28.28, - 23.87, 15.17001, 19.38, 7.420013, 25.79001, 11.75, 19.97, 17.08002, - 30.25, 16.03, 15.5, 14.81, 9.410004, 30.12, 29.95001, 28.56 ; - } // group ObsValue - -group: PreQC { - variables: - float salinity(Location) ; - salinity:_FillValue = 9.96921e+36f ; - float waterTemperature(Location) ; - waterTemperature:_FillValue = 9.96921e+36f ; - data: - - salinity = 0.08789432, 0.08789432, 0.08789432, 0.08789432, 0.08789432, - 0.08789432, 0.08789432, 0.08789432, 0.08789432, 0.08789432, 0.08789432, - 0.2789434, 0.2789434, 0.2789434, 0.2789434, 0.2789434, 0.2789434, - 0.2789434, 0.2789434, 0.2789434, 0.2789434, 0.4881761, 0.4881761, - 0.4881761, 0.4881761, 0.4881761, 0.4881761, 0.4881761, 0.4881761, - 0.4881761, 0.4881761, 0.4881761, 0.4881761, 0.09344757, 0.09344757, - 0.09344757, 0.09344757, 0.09344757, 0.09344757, 0.09344757, 0.09344757, - 0.09344757, 0.09344757, 0.3562652, 0.3562652, 0.3562652, 0.3562652, - 0.3562652, 0.3562652, 0.3562652, 0.3562652, 0.3562652, 0.3562652, - 0.3562652, 0.6465657, 0.6465657, 0.6465657, 0.6465657, 0.6465657, - 0.6465657, 0.6465657, 0.6465657, 0.6465657, 0.245206, 0.245206, - 0.245206, 0.245206, 0.245206, 0.245206, 0.245206, 0.245206, 0.245206, - 0.344362, 0.344362, 0.344362, 0.344362, 0.344362, 0.344362, 0.344362, - 0.344362, 0.344362, 0.344362, 0.344362, 0.344362, 0.2459739, 0.2459739, - 0.2459739, 0.2459739, 0.2459739, 0.2459739, 0.2459739, 0.2459739, - 0.2459739, 0.2459739, 0.2994778, 0.2994778, 0.2994778, 0.2994778, - 0.2994778, 0.2994778, 0.2994778, 0.2994778, 0.2994778, 0.2994778, - 0.2994778, 0.3914765, 0.3914765, 0.3914765, 0.3914765, 0.3914765, - 0.3914765, 0.3914765, 0.3914765, 0.3914765, 0.3914765, 0.6239262, - 0.6239262, 0.6239262, 0.6239262, 0.6239262, 0.6239262, 0.6239262, - 0.6239262, 0.6239262, 0.6239262, 0.6239262, 0.6239262, 0.6239262, - 0.1783193, 0.1783193, 0.1783193, 0.1783193, 0.1783193, 0.1783193, - 0.1783193, 0.1783193, 0.1783193, 0.1653826, 0.1653826, 0.1653826, - 0.1653826, 0.1653826, 0.1653826, 0.1653826, 0.1653826, 0.1653826, - 0.1653826, 0.1653826, 0.5274251, 0.5274251, 0.5274251, 0.5274251, - 0.5274251, 0.5274251, 0.5274251, 0.5274251, 0.5274251, 0.5274251, - 0.3551692, 0.3551692, 0.3551692, 0.3551692, 0.3551692, 0.3551692, - 0.3551692, 0.3551692, 0.3551692, 0.3551692, 0.3551692, 0.2711022, - 0.2711022, 0.2711022, 0.2711022, 0.2711022, 0.2711022, 0.2711022, - 0.2711022, 0.2711022, 0.2711022, 0.4063673, 0.4063673, 0.4063673, - 0.4063673, 0.4063673, 0.4063673, 0.4063673, 0.4063673, 0.4063673, - 0.4063673, 0.4063673, 0.4063673, 0.4063673, 0.3125914, 0.3125914, - 0.3125914, 0.3125914, 0.3125914, 0.3125914, 0.3125914, 0.3125914, - 0.3125914, 0.3125914, 0.3125914, 0.3125914, 0.3125914, 0.3125914, - 0.4828305, 0.4828305, 0.4828305, 0.4828305, 0.4828305, 0.4828305, - 0.4828305, 0.4828305, 0.4828305, 0.4828305, 0.4828305, 0.4828305 ; - - waterTemperature = 0.3304284, 0.3304284, 0.3304284, 0.3304284, 0.3304284, - 0.3304284, 0.3304284, 0.3304284, 0.3304284, 0.3304284, 0.3304284, - 0.4522219, 0.4522219, 0.4522219, 0.4522219, 0.4522219, 0.4522219, - 0.4522219, 0.4522219, 0.4522219, 0.4522219, 0.5740165, 0.5740165, - 0.5740165, 0.5740165, 0.5740165, 0.5740165, 0.5740165, 0.5740165, - 0.5740165, 0.5740165, 0.5740165, 0.5740165, 0.3443506, 0.3443506, - 0.3443506, 0.3443506, 0.3443506, 0.3443506, 0.3443506, 0.3443506, - 0.3443506, 0.3443506, 0.2669469, 0.2669469, 0.2669469, 0.2669469, - 0.2669469, 0.2669469, 0.2669469, 0.2669469, 0.2669469, 0.2669469, - 0.2669469, 0.5118217, 0.5118217, 0.5118217, 0.5118217, 0.5118217, - 0.5118217, 0.5118217, 0.5118217, 0.5118217, 0.4708711, 0.4708711, - 0.4708711, 0.4708711, 0.4708711, 0.4708711, 0.4708711, 0.4708711, - 0.4708711, 0.3297694, 0.3297694, 0.3297694, 0.3297694, 0.3297694, - 0.3297694, 0.3297694, 0.3297694, 0.3297694, 0.3297694, 0.3297694, - 0.3297694, 0.7652588, 0.7652588, 0.7652588, 0.7652588, 0.7652588, - 0.7652588, 0.7652588, 0.7652588, 0.7652588, 0.7652588, 0.399832, - 0.399832, 0.399832, 0.399832, 0.399832, 0.399832, 0.399832, 0.399832, - 0.399832, 0.399832, 0.399832, 0.4094992, 0.4094992, 0.4094992, - 0.4094992, 0.4094992, 0.4094992, 0.4094992, 0.4094992, 0.4094992, - 0.4094992, 0.7739636, 0.7739636, 0.7739636, 0.7739636, 0.7739636, - 0.7739636, 0.7739636, 0.7739636, 0.7739636, 0.7739636, 0.7739636, - 0.7739636, 0.7739636, 0.5043824, 0.5043824, 0.5043824, 0.5043824, - 0.5043824, 0.5043824, 0.5043824, 0.5043824, 0.5043824, 0.4389331, - 0.4389331, 0.4389331, 0.4389331, 0.4389331, 0.4389331, 0.4389331, - 0.4389331, 0.4389331, 0.4389331, 0.4389331, 0.3698398, 0.3698398, - 0.3698398, 0.3698398, 0.3698398, 0.3698398, 0.3698398, 0.3698398, - 0.3698398, 0.3698398, 0.2308534, 0.2308534, 0.2308534, 0.2308534, - 0.2308534, 0.2308534, 0.2308534, 0.2308534, 0.2308534, 0.2308534, - 0.2308534, 0.5125766, 0.5125766, 0.5125766, 0.5125766, 0.5125766, - 0.5125766, 0.5125766, 0.5125766, 0.5125766, 0.5125766, 0.3485442, - 0.3485442, 0.3485442, 0.3485442, 0.3485442, 0.3485442, 0.3485442, - 0.3485442, 0.3485442, 0.3485442, 0.3485442, 0.3485442, 0.3485442, - 0.3094093, 0.3094093, 0.3094093, 0.3094093, 0.3094093, 0.3094093, - 0.3094093, 0.3094093, 0.3094093, 0.3094093, 0.3094093, 0.3094093, - 0.3094093, 0.3094093, 0.5460482, 0.5460482, 0.5460482, 0.5460482, - 0.5460482, 0.5460482, 0.5460482, 0.5460482, 0.5460482, 0.5460482, - 0.5460482, 0.5460482 ; - } // group PreQC -} diff --git a/test/soca/testdata/sss.nc.cdl b/test/soca/testdata/sss.nc.cdl deleted file mode 100644 index 7c943171b..000000000 --- a/test/soca/testdata/sss.nc.cdl +++ /dev/null @@ -1,153 +0,0 @@ -netcdf sss.nc { -dimensions: - Location = 100 ; - nvars = 1 ; -variables: - int Location(Location) ; - Location:suggested_chunk_dim = 100LL ; - float nvars(nvars) ; - nvars:suggested_chunk_dim = 100LL ; - -// global attributes: - string :_ioda_layout = "ObsGroup" ; - :_ioda_layout_version = 0 ; - :nrecs = 1 ; - :nvars = 1 ; - :odb_version = 1LL ; - :date_time = 2018041512 ; - :nlocs = 100LL ; - :nobs = 100LL ; - :history = "Fri Apr 19 08:41:14 2019: ncks -d nlocs,1,1000000,10000 sss.orig.nc -O sst.nc" ; - :NCO = "4.7.2" ; -data: - - Location = 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0 ; - - nvars = 0 ; - -group: MetaData { - variables: - int64 dateTime(Location) ; - dateTime:_FillValue = -3732782400LL ; - string dateTime:units = "seconds since 2018-04-15T12:00:00Z" ; - string date_time(Location) ; - string date_time:_FillValue = "" ; - float latitude(Location) ; - latitude:_FillValue = 9.96921e+36f ; - float longitude(Location) ; - longitude:_FillValue = 9.96921e+36f ; - int sequenceNumber(Location) ; - sequenceNumber:_FillValue = -2147483647 ; - string variable_names(nvars) ; - string variable_names:_FillValue = "" ; - data: - - dateTime = 16413, -15923, -37695, 14766, -25781, -11737, -9986, 20459, - -25203, -18263, 36958, 21046, -11400, -23577, -32343, -4229, -19814, - -19361, 9764, 22504, -19662, 1093, -37076, 22639, -38226, 38470, 3355, - -21918, -28049, 32159, -1882, 6817, 28001, 5033, -18020, -33747, 25053, - 42188, -15155, 36241, 761, -31254, -15877, 7592, 20573, -8920, 29349, - 29808, 21036, 31289, -10162, 11895, -22210, 12609, 27289, 2940, 32325, - -23540, 7420, 21173, -37473, 25112, -11645, -3995, 23491, -30758, - 24175, -34669, 27365, 21227, 43137, -5626, -15018, -25329, -12130, - -12186, -32019, -16964, 12260, -19083, 2815, -31577, -11805, 27481, - -17853, -38072, 29789, -8915, 43012, -26812, -27762, -33971, -20603, - 3250, -20704, 42063, 42910, 9413, -37307, -31455 ; - - latitude = 53.879, -58.38232, 0.393135, -46.62201, 0.3569438, 37.38496, - -58.87528, -64.11797, 32.36138, 75.8694, -49.12336, -27.39389, 25.3784, - 39.12261, -34.36073, -50.38835, 0.884598, 28.35575, 6.134513, 63.87992, - 10.13324, -19.62038, 35.38895, 66.38779, -39.14023, -47.11112, - -18.87907, -61.36031, -41.6361, -61.61643, 13.61734, -10.87163, - 34.38682, 72.36931, 60.62663, -60.38211, -42.64268, -6.871958, - -64.62761, -6.131754, -2.378339, 24.61968, -61.36315, -59.37663, - -57.13705, -50.12148, 53.11937, 34.37182, -20.87342, -59.36716, - -57.62246, 37.1406, -34.88064, 2.888079, -8.607044, -52.12254, - -57.8832, 37.8768, -43.11407, -12.37021, 12.12483, -38.37106, 40.37134, - -65.13678, 58.61689, 53.35913, 9.118921, -4.877728, 3.642554, - -18.13534, -56.38377, 27.12191, -57.11341, 32.64145, 60.38355, - 70.39142, -14.36325, 5.119036, 15.39068, 45.37792, -56.89023, 11.36977, - 41.8684, 8.370608, 50.36518, -29.38099, 28.11311, -45.87344, -49.12109, - -63.39182, -65.10889, -44.85678, -46.86501, -30.36576, -43.63494, - -0.8765696, -51.63343, -9.134258, 20.61064, 20.11235 ; - - longitude = 4.115798, 327.8615, 248.3797, 44.38053, 201.8728, 324.8867, - 294.0996, 17.3724, 188.1367, 43.91171, 110.8501, 10.37052, 318.6276, - 18.88903, 227.636, 279.3855, 172.6065, 165.1211, 52.86695, 330.1345, - 170.1191, 258.1304, 234.6186, 333.8995, 251.132, 297.3483, 76.89408, - 348.4011, 25.36266, 332.1636, 94.8727, 236.8727, 327.3827, 32.67982, - 355.6476, 36.36999, 154.8804, 97.62099, 179.1315, 119.6152, 270.8696, - 213.1319, 327.116, 230.1518, 18.37139, 144.6165, 160.592, 153.3711, - 8.64412, 117.3863, 304.6573, 227.6308, 354.8997, 216.3867, 338.3861, - 86.59863, 336.36, 19.87214, 237.3763, 3.612482, 241.6204, 158.8994, - 327.3783, 257.3757, 188.6524, 206.8672, 170.8689, 53.62115, 333.1209, - 4.131203, 78.86398, 294.1184, 163.8975, 187.8797, 338.088, 353.3509, - 229.1338, 340.6393, 220.6252, 157.6046, 82.86684, 217.1175, 331.1054, - 330.3774, 352.1118, 257.6353, 147.6305, 133.653, 77.65506, 223.3561, - 11.3523, 38.86535, 183.6522, 76.38167, 186.3695, 96.36871, 85.86997, - 51.61628, 238.3724, 217.1181 ; - - sequenceNumber = 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1 ; - - variable_names = "sea_surface_salinity" ; - } // group MetaData - -group: ObsError { - variables: - float seaSurfaceSalinity(Location) ; - seaSurfaceSalinity:_FillValue = 9.96921e+36f ; - data: - - seaSurfaceSalinity = 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ; - } // group ObsError - -group: ObsValue { - variables: - float seaSurfaceSalinity(Location) ; - seaSurfaceSalinity:_FillValue = 9.96921e+36f ; - data: - - seaSurfaceSalinity = 33.71178, 34.27895, 34.89477, 33.47415, 35.26365, - 35.16874, 34.30404, 34.09843, 34.96246, 0.007955075, 35.2322, 35.57863, - 37.57914, 38.05683, 34.03303, 33.31181, 33.65787, 35.14495, 36.04657, - 34.5812, 34.02576, 36.25037, 32.16301, 34.79661, 34.63808, 33.19758, - 34.36973, 36.37263, 35.16792, 33.12465, 32.51355, 35.36753, 36.12861, - 34.40482, 35.7425, 35.411, 35.9472, 34.48295, 32.57633, 25.14996, - 34.82872, 34.71701, 35.64753, 34.48979, 34.17043, 32.8913, 0.0047155, - 35.08466, 35.65135, 32.68414, 34.3378, 33.15457, 35.72908, 34.25256, - 36.46022, 32.59492, 33.59892, 34.7174, 33.43561, 36.39915, 34.19456, - 35.34409, 36.00577, 32.72109, 31.62992, 33.06574, 33.46413, 35.41788, - 35.51361, 36.36302, 33.78024, 37.16262, 32.05506, 34.65675, 35.83234, - 34.34883, 36.18774, 33.99451, 33.84148, 32.32932, 36.14341, 33.85884, - 35.04386, 36.04567, 36.23658, 36.13433, 34.53632, 31.36066, 33.58314, - 33.99441, 35.13243, 33.27727, 35.24531, 36.08514, 35.2952, 31.50171, - 34.44715, 35.18509, 34.1835, 35.66429 ; - } // group ObsValue - -group: PreQC { - variables: - int seaSurfaceSalinity(Location) ; - seaSurfaceSalinity:_FillValue = -2147483647 ; - data: - - seaSurfaceSalinity = 8224, 2048, 0, 4096, 0, 0, 128, 2048, 0, 28192, 0, - 8192, 0, 8192, 0, 8192, 8192, 0, 8192, 8224, 8192, 0, 8224, 27136, 0, - 8192, 0, 2048, 0, 2048, 40960, 0, 0, 27136, 8192, 2048, 0, 0, 18432, - 9472, 0, 0, 2048, 2048, 6144, 0, 28416, 0, 8192, 2048, 2048, 0, 0, 0, - 0, 0, 2048, 8448, 0, 0, 32, 0, 32768, 18432, 14336, 0, 8192, 0, 32768, - 0, 2048, 0, 0, 0, 0, 59392, 0, 32768, 128, 2048, 2048, 32, 0, 32, 8192, - 0, 0, 4096, 128, 2048, 6144, 0, 0, 0, 0, 40960, 0, 0, 32, 0 ; - } // group PreQC -} diff --git a/test/soca/testdata/sst.nc.cdl b/test/soca/testdata/sst.nc.cdl deleted file mode 100644 index e61c2e11d..000000000 --- a/test/soca/testdata/sst.nc.cdl +++ /dev/null @@ -1,203 +0,0 @@ -netcdf sst.nc { -dimensions: - Location = 201 ; -variables: - int Location(Location) ; - Location:suggested_chunk_dim = 201LL ; - -// global attributes: - string :_ioda_layout = "ObsGroup" ; - :_ioda_layout_version = 0 ; - :nrecs = 1 ; - :converter = "gds2_sst2ioda.py" ; - :processing_level = "L3U" ; - :nlocs = 201 ; - :nobs = 201 ; - :thinning = 0.99999 ; - :sensor = "AVHRR_GAC" ; - :platform = "NOAA-19" ; -data: - - Location = 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ; - - -group: MetaData { - variables: - int64 dateTime(Location) ; - dateTime:_FillValue = -3732782400LL ; - string dateTime:units = "seconds since 2018-04-15T12:00:00Z" ; - float latitude(Location) ; - latitude:_FillValue = 9.96921e+36f ; - float longitude(Location) ; - longitude:_FillValue = 9.96921e+36f ; - - data: - - dateTime = -40813, -41195, -41918, -41960, -42007, -42014, -42017, -36233, - -34897, -33232, -35478, -35689, -35678, -35735, -35715, -35743, -35785, - -35793, -35785, -35883, -35908, -29089, -29166, -29200, -29259, -29550, - -29582, -29905, -30064, -32128, -31982, -26780, -26555, -26398, -26400, - -26190, -26095, -25842, -25787, -22996, -23045, -23068, -23191, -23338, - -23439, -23523, -23903, -23962, -21107, -21068, -20959, -20775, -20786, - -20717, -20347, -19585, -14862, -14722, -16968, -17083, -17174, -17163, - -17576, -17598, -17587, -10919, -11331, -11437, -11469, -8438, -8172, - -8145, -7703, -7664, -7472, -7230, -3743, -3867, -4812, -5357, -5363, - -2328, -2212, -2116, -1825, -1820, -1681, -1671, -1683, -1682, -1564, - 2231, 3179, 870, 826, 3842, 3926, 3954, 4056, 4074, 4194, 4243, 4264, - 4388, 4518, 9214, 9281, 10234, 10274, 10252, 10296, 10590, 10635, - 10608, 10667, 14008, 13880, 13869, 13773, 13199, 12874, 12817, 12636, - 12659, 10827, 12394, 15350, 15513, 15596, 16616, 16664, 16851, 17139, - 17170, 20450, 20414, 20423, 19972, 19922, 19871, 19825, 19825, 19635, - 19398, 19281, 19262, 19225, 19178, 19013, 18943, 18893, 18909, 18263, - 21885, 22125, 22658, 22794, 24800, 27876, 25968, 25956, 28112, 28166, - 25762, 28718, 32111, 32144, 32008, 31987, 31902, 31528, 28826, 28868, - 30927, 30882, 34355, 34414, 34928, 34986, 35036, 35499, 38222, 38147, - 38096, 37996, 37982, 37870, 37785, 37711, 37597, 37596, 37539, 37510, - 36943, 40469, 40520, 40518, 41066, 41181, 41548, 41722 ; - - latitude = 58.91, 36.77, -5.189998, -8.749998, -9.209998, -11.37, -12.21, - -30.81, 44.97, 31.83, 12.69, 1.890002, 1.550002, -0.449998, -1.009998, - -1.069998, -3.889998, -4.049998, -5.009998, -9.989998, -11.23, 27.99, - 25.07, 21.37, 18.37, 0.430002, -0.689998, -19.11, -28.89, -30.09, - -37.57, 15.45, -0.02999799, -7.609998, -8.089998, -21.83, -25.87, - -41.25, -44.51, 28.63, 24.47, 21.53, 17.03, 5.910002, 0.330002, - -3.189998, -24.83, -27.81, 40.85, 37.27, 32.95, 21.43, 20.71, 17.75, - -5.669998, -47.19, 31.81, 23.51, 23.27, 16.43, 9.950002, 9.130002, - -12.39, -13.63, -15.51, 16.33, -4.809998, -10.91, -13.17, 14.95, - -0.669998, -1.689998, -28.69, -30.87, -40.93, -56.17, 75.23, 69.91, - 15.65, -14.41, -15.29, 16.13, 10.39, 2.850002, -13.75, -14.19, -20.07, - -21.29, -22.25, -22.33, -28.95, 66.39, 53.09, -9.149998, -11.75, 13.47, - 8.850002, 6.530002, 0.930002, -0.149998, -7.929998, -10.55, -11.01, - -19.73, -27.47, 56.17, 51.93, -2.429998, -4.749998, -5.109998, - -5.869998, -23.33, -24.47, -24.79, -26.97, 42.07, 36.67, 36.01, 28.47, - -3.769998, -21.71, -25.51, -35.57, -35.81, -37.35, -50.17, 55.21, - 47.49, 41.33, -17.57, -21.37, -31.09, -47.69, -50.07, 62.31, 60.27, - 59.67, 35.89, 32.99, 30.05, 26.29, 25.37, 16.09, 1.590002, -5.749998, - -6.289998, -10.31, -12.45, -20.11, -24.85, -26.97, -26.99, -63.71, - 31.59, 17.41, -13.87, -22.57, -41.61, 39.39, 27.23, 24.99, 23.49, - 21.55, 13.71, -10.79, 29.19, 28.51, 23.43, 22.27, 17.45, -7.489998, - -16.65, -17.43, -39.65, -42.59, 19.69, 15.79, -14.25, -17.41, -22.57, - -47.19, 27.01, 25.01, 20.03, 15.27, 13.89, 6.230002, 3.550002, - -1.489998, -8.429997, -8.469997, -10.93, -12.59, -45.69, 19.47, 16.73, - 16.69, -16.89, -20.33, -42.73, -54.77 ; - - longitude = -146.77, -138.63, -127.65, -134.05, -119.81, -129.79, -134.81, - -148.69, -176.33, 31.52999, -161.01, -149.97, -155.71, -147.33, - -157.67, -148.01, -149.67, -147.95, -156.25, -150.59, -148.97, 168.45, - 177.67, 169.21, 172.01, 170.07, 174.41, -179.27, 177.51, 1.809996, - -5.130004, -17.29, -3.950004, -16.79, -12.55, -3.790004, -17.27, - -15.91, -15.99, 156.91, 148.61, 141.31, 157.43, 144.53, 147.39, 156.53, - 164.27, 167.61, -30.03, -22.47, -40.95, -34.53, -26.09, -33.51, -26.81, - -56.07, -46.83, -47.95, 132.81, 133.65, 126.99, 118.89, 138.61, 139.15, - 122.17, 92.50999, 113.47, 115.09, 113.71, -80.95, -83.95, -88.07, - -84.49001, -85.51, -98.15, -84.15, 25.67, 42.94999, 67.81, 84.34999, - 81.09, -109.67, -118.61, -106.95, -113.31, -112.51, -127.25, -124.17, - -113.89, -113.63, -116.77, 10.91, -140.25, 52.02999, 52.39, -136.85, - -139.79, -136.03, -139.51, -139.85, -136.21, -138.79, -143.83, -135.37, - -134.95, -143.37, -143.01, -165.79, -166.51, -155.11, -168.03, -170.13, - -178.57, -167.83, -176.11, -15.97, -3.190004, -3.130004, -11.81, - 1.829996, 11.89, 10.11, 15.29, 3.549996, -172.33, 15.91, -168.73, - 174.71, -176.05, 164.31, 170.67, 159.81, 154.63, 157.43, -40.59, - -39.29, -46.19, -19.45, -18.89, -18.31, -26.91, -31.93, -18.51, -19.89, - -21.11, -17.79, -29.09, -24.25, -10.85, -13.43, -8.810004, -13.93, - 2.669996, 155.27, 152.85, 145.99, 151.13, -45.35, 130.49, -54.45, - -62.13, 137.37, 131.07, -59.69, 125.01, -75.91, -90.23, -72.83, - -72.35001, -70.21, -85.07, 120.31, 109.17, -59.59, -60.53, 90.78999, - 93.46999, 86.74999, 84.73, 100.71, 76.38999, -110.47, -93.75, -107.01, - -99.67, -102.53, -107.39, -93.09, -96.47, -96.41, -96.37, -90.85, - -90.49001, -82.77, 70.41, 67.73, 69.17, 72.31, 51.64999, 50.84999, 61.07 ; - - } // group MetaData - -group: ObsError { - variables: - float seaSurfaceTemperature(Location) ; - seaSurfaceTemperature:_FillValue = 9.96921e+36f ; - string seaSurfaceTemperature:units = "c" ; - - data: - - seaSurfaceTemperature = 0.29, 0.29, 0.39, 0.47, 0.39, 0.39, 0.42, 0.37, - 0.32, 0.39, 0.39, 0.4, 0.39, 0.42, 0.39, 0.39, 0.34, 0.37, 0.39, 0.48, - 0.48, 0.29, 0.27, 0.33, 0.43, 0.48, 0.58, 0.39, 0.36, 0.22, 0.28, 0.4, - 0.54, 0.28, 0.29, 0.32, 0.24, 0.27, 0.28, 0.29, 0.32, 0.42, 0.43, 0.48, - 0.63, 0.48, 0.32, 0.36, 0.27, 0.26, 0.31, 0.36, 0.26, 0.21, 0.25, 0.29, - 0.23, 0.24, 0.37, 0.39, 0.44, 0.52, 0.48, 0.48, 0.48, 0.53, 0.56, 0.51, - 0.45, 0.34, 0.3, 0.19, 0.24, 0.29, 0.27, 0.42, 0.33, 0.33, 0.53, 0.39, - 0.41, 0.23, 0.33, 0.29, 0.19, 0.25, 0.34, 0.36, 0.19, 0.21, 0.27, 0.39, - 0.29, 0.44, 0.48, 0.32, 0.48, 0.39, 0.27, 0.17, 0.2, 0.41, 0.26, 0.44, - 0.32, 0.28, 0.28, 0.29, 0.3, 0.33, 0.46, 0.14, 0.34, 0.22, 0.34, 0.29, - 0.29, 0.29, 0.43, 0.48, 0.29, 0.29, 0.28, 0.31, 0.25, 0.28, 0.28, 0.33, - 0.29, 0.42, 0.27, 0.31, 0.27, 0.28, 0.32, 0.28, 0.29, 0.36, 0.34, 0.45, - 0.29, 0.35, 0.34, 0.48, 0.45, 0.45, 0.46, 0.39, 0.33, 0.32, 0.35, 0.3, - 0.3, 0.24, 0.3, 0.22, 0.27, 0.3, 0.28, 0.3, 0.39, 0.27, 0.33, 0.52, - 0.22, 0.29, 0.36, 0.36, 0.42, 0.48, 0.48, 0.46, 0.53, 0.27, 0.29, 0.52, - 0.49, 0.26, 0.23, 0.34, 0.32, 0.44, 0.38, 0.44, 0.48, 0.48, 0.52, 0.5, - 0.35, 0.31, 0.31, 0.39, 0.36, 0.29, 0.51, 0.45, 0.5, 0.3, 0.32, 0.27, - 0.21 ; - } // group ObsError - -group: ObsValue { - variables: - float seaSurfaceTemperature(Location) ; - seaSurfaceTemperature:_FillValue = 9.96921e+36f ; - string seaSurfaceTemperature:units = "c" ; - - data: - - seaSurfaceTemperature = 5.803994, 13.77201, 27.064, 27.87, 26.804, 27.56, - 28.04401, 23.72399, 5.648, 18.796, 27.22001, 27.57401, 27.27401, - 26.31599, 26.88601, 26.71201, 27.69399, 27.76599, 27.99201, 29.282, - 29.158, 20.49, 22.51001, 25.30799, 26.21601, 29.93198, 28.80801, - 27.218, 23.48199, 21.332, 17.88, 18.09201, 28.69399, 28.20799, - 27.52001, 23.73, 25.448, 16.22801, 11.512, 19.95001, 25.152, 25.94599, - 27.108, 30.13799, 30.17001, 30.59599, 25.03801, 23.708, 15.966, - 17.34999, 20.70201, 22.19199, 20.86601, 22.85399, 28.68199, 12.99401, - 21.56801, 23.868, 26.36601, 28.03, 29.3, 29.282, 30.19199, 30.42599, - 30.96801, 29.74399, 29.97799, 30.94201, 30.21599, 27.03999, 21.54199, - 24.39801, 21.08001, 21.15201, 14.18999, 6.417985, 0.9120085, 1.09599, - 29.56199, 27.56801, 27.42999, 27.35798, 27.83399, 28.49199, 25.51601, - 25.26, 26.60399, 26.15399, 25.008, 25.056, 25.03201, 6.167985, - 6.300011, 29.50001, 28.94001, 25.94, 27.66201, 27.76801, 25.988, - 26.44201, 27.26599, 28.226, 28.69601, 27.536, 25.972, 5.996015, - 5.86601, 27.63, 28.854, 27.80999, 29.11199, 26.074, 24.92398, 24.782, - 24.25601, 13.10801, 15.238, 15.362, 17.10798, 31.26999, 20.30999, - 20.714, 19.51001, 19.93201, 19.314, 4.237989, 4.211993, 3.711993, - 9.389996, 27.23799, 25.79201, 23.59601, 11.88599, 10.11001, 3.012011, - 4.411992, -1.927994, 17.23599, 18.82001, 19.76399, 21.04601, 21.82199, - 21.10999, 29.21801, 28.53401, 28.57801, 28.14401, 27.05801, 24.99401, - 25.41801, 24.67, 25.43999, 0.08600003, 19.12601, 27.44599, 28.01801, - 25.87999, 16.278, 8.505994, 24.53199, 25.18201, 24.6, 26.978, 27.11801, - 30.42799, 23.09, 22.23598, 25.93399, 26.22599, 27.09799, 25.372, - 30.39801, 27.93799, 17.91599, 15.72001, 28.846, 29.122, 27.20201, - 26.73599, 23.46399, 7.969999, 20.43399, 24.03599, 24.56201, 30.00001, - 29.84599, 29.97601, 28.97599, 26.50001, 27.156, 27.18799, 25.97801, - 25.40001, 11.42001, 28.83, 29.22801, 29.294, 27.58001, 27.64801, - 10.55799, 3.594002 ; - } // group ObsValue - -group: PreQC { - variables: - float seaSurfaceTemperature(Location) ; - seaSurfaceTemperature:_FillValue = 9.96921e+36f ; - data: - - seaSurfaceTemperature = 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ; - } // group PreQC -} diff --git a/ush/examples/run_scripts/soca025.yaml b/ush/examples/run_scripts/soca025.yaml index 9acfe1b0e..e1a7c4541 100644 --- a/ush/examples/run_scripts/soca025.yaml +++ b/ush/examples/run_scripts/soca025.yaml @@ -31,8 +31,5 @@ jedi static: model background: ocn: /work2/noaa/da/gvernier/data/test_bkg/1440x1080x75/ -r2d2 options: - root: /work2/noaa/da/gvernier/data/static/r2d2-shared - fms: domain_stack_size: 5832000 diff --git a/ush/r2d2/example_store_bkg.py b/ush/r2d2/example_store_bkg.py deleted file mode 100644 index 8909a026a..000000000 --- a/ush/r2d2/example_store_bkg.py +++ /dev/null @@ -1,46 +0,0 @@ -import os -from solo.configuration import Configuration -import ufsda.r2d2 -import yaml - - -def store_bkg(yamlfile): - config = Configuration(yamlfile) - ufsda.r2d2.store(config) - - -if __name__ == "__main__": - # first create dict of config for UFS RESTART files - config = { - 'start': '2021-12-25T00:00:00Z', - 'end': '2021-12-27T00:00:00Z', - 'step': 'PT6H', - 'forecast_steps': ['PT6H'], - 'file_type_list': ['fv_core.res', 'fv_srf_wnd.res', 'fv_tracer.res', 'phy_data', 'sfc_data'], - 'source_dir': '/work2/noaa/da/rtreadon/data/', - 'source_file_fmt': '{source_dir}/enkf{dump}.{year}{month}{day}/{hour}/atmos/\ -mem001/RESTART/$(valid_date).$(file_type).tile$(tile).nc', - 'type': 'fc', - 'model': 'gfs', - 'resolution': 'c384', - 'database': 'shared', - 'dump': 'gdas', - 'experiment': 'oper_gdas', - 'tile': [1, 2, 3, 4, 5, 6], - 'user_date_format': '%Y%m%d.%H%M%S', - 'fc_date_rendering': 'analysis', - } - yamlfile = os.path.join(os.getcwd(), 'store_gfs_bkg.yaml') - with open(yamlfile, 'w') as f: - yaml.dump(config, f, sort_keys=False, default_flow_style=False) - store_bkg(yamlfile) - # replace gfs with gfs_metadata - config['model'] = 'gfs_metadata' - config['source_file_fmt'] = '{source_dir}/enkf{dump}.{year}{month}{day}/{hour}/atmos/\ -mem001/RESTART/$(valid_date).$(file_type)' - config['file_type_list'] = ['coupler.res', 'fv_core.res.nc'] - del config['tile'] - yamlfile = os.path.join(os.getcwd(), 'store_gfs_coupler.yaml') - with open(yamlfile, 'w') as f: - yaml.dump(config, f, sort_keys=False, default_flow_style=False) - store_bkg(yamlfile) diff --git a/ush/r2d2/proc_ops2r2d2.sh b/ush/r2d2/proc_ops2r2d2.sh deleted file mode 100755 index 3857d4ca1..000000000 --- a/ush/r2d2/proc_ops2r2d2.sh +++ /dev/null @@ -1,279 +0,0 @@ -#!/bin/bash -diagroot=/work2/noaa/stmp/cmartin/obs/ -extract="NO" -iodaconv="NO" -satbiasconv="NO" -r2d2store_obs="NO" -r2d2store_bc="YES" -startdate=2021122100 -enddate=2021123118 -dump="gdas" -HOMEgdas=/work2/noaa/da/cmartin/GDASApp/work/GDASApp -machine="orion" - -#--------------------------------------------------------------- -# do not modify below here -#--------------------------------------------------------------- - -# load runtime env -#--------------------------------------------------------------- -module purge -module use $HOMEgdas/modulefiles -module load GDAS/$machine - -# extract from tar files -#--------------------------------------------------------------- -if [ $extract = "YES" ]; then - diags="oznstat cnvstat radstat" - nowdate=$startdate - while test $nowdate -le $enddate ; do - echo "Extracting $nowdate" - nowpdy=${nowdate::8}; nowcyc=${nowdate:8} - diagdir=$diagroot/${dump}.$nowpdy/$nowcyc/atmos/ - cd $diagdir - for d in $diags; do - tar xvf ${dump}.t??z.$d - done - for f in $(ls diag*gz); do - gunzip -f $f - rm -rf $f - done - mkdir -p diags - rm -rf diag*anl* - mv diag*ges* diags/. - nowdate=$(date -d "$nowpdy $nowcyc + 6 hour" +%Y%m%d%H) - done -fi - -# run ioda converters -#--------------------------------------------------------------- -convertpy=$HOMEgdas/build/bin/proc_gsi_ncdiag.py -combinepy=$HOMEgdas/build/bin/combine_obsspace.py -export PYTHONPATH=$PYTHONPATH:$HOMEgdas/build/lib/python3.7/pyioda - -if [ $iodaconv = "YES" ]; then - nowdate=$startdate - while test $nowdate -le $enddate ; do - echo "Converting obs to IODA for $nowdate" - nowpdy=${nowdate::8}; nowcyc=${nowdate:8} - diagdir=$diagroot/${dump}.$nowpdy/$nowcyc/atmos/ - cd $diagdir - mkdir -p $diagdir/ioda - $convertpy -o $diagdir/ioda $diagdir/diags - $combinepy -i $diagdir/ioda/sfc_*.nc4 -o $diagdir/ioda/sfc_obs_"$nowdate".nc4 - $combinepy -i $diagdir/ioda/sfcship_*.nc4 -o $diagdir/ioda/sfcship_obs_"$nowdate".nc4 - $combinepy -i $diagdir/ioda/sondes_*.nc4 -o $diagdir/ioda/sondes_obs_"$nowdate".nc4 - $combinepy -i $diagdir/ioda/aircraft_*.nc4 -o $diagdir/ioda/aircraft_obs_"$nowdate".nc4 - nowdate=$(date -d "$nowpdy $nowcyc + 6 hour" +%Y%m%d%H) - done -fi - -# run satbias converters -#--------------------------------------------------------------- -satbiaspy=$HOMEgdas/ush/run_satbias_conv.py -if [ $satbiasconv = "YES" ]; then - # python script runs over all period - nowdate=$startdate - nowpdy=${nowdate::8}; nowcyc=${nowdate:8} - prevdate=$(date -d "$nowpdy $nowcyc - 6 hour" +%Y-%m-%dT%H) - lastpdy=${enddate::8}; lastcyc=${enddate:8} - lastprevdate=$(date -d "$lastpdy $lastcyc - 6 hour" +%Y-%m-%dT%H) - # create YAML file - cd $diagroot - cat > $diagroot/run_satbias_conv.yaml << EOF -start time: $prevdate:00:00Z -end time: $lastprevdate:00:00Z -assim_freq: 6 -gsi_bc_root: $diagroot -ufo_bc_root: $diagroot/satbias -work_root: $diagroot/work -satbias2ioda: $HOMEgdas/build/bin/satbias2ioda.x -dump: $dump -EOF - $satbiaspy --config $diagroot/run_satbias_conv.yaml -fi - -# store with r2d2 -#--------------------------------------------------------------- -r2d2storepy=$HOMEgdas/ush/r2d2/r2d2_store.py -export PYTHONPATH=$PYTHONPATH:$HOMEgdas/ush - -if [ $r2d2store_obs = "YES" ]; then - cd $diagroot - nowpdy=${startdate::8}; nowcyc=${startdate:8} - firstdate=$(date -d "$nowpdy $nowcyc" +%Y-%m-%dT%H) - nowpdy=${enddate::8}; nowcyc=${enddate:8} - lastdate=$(date -d "$nowpdy $nowcyc" +%Y-%m-%dT%H) - cat > $diagroot/r2d2_store_obs.yaml << EOF -start: ${firstdate}:00:00Z -end: ${lastdate}:00:00Z -step: PT6H -source_dir: $diagroot -source_file_fmt: '{source_dir}/{dump}.{year}{month}{day}/{hour}/atmos/ioda/{obs_type}_obs_{year}{month}{day}{hour}.nc4' -type: ob -database: shared -provider: ncdiag -experiment: oper_$dump -obs_types: - - aircraft - - amsua_metop-a - - amsua_metop-b - - amsua_metop-c - - amsua_n15 - - amsua_n18 - - amsua_n19 - - atms_n20 - - atms_npp - - avhrr3_metop-a - - avhrr3_n18 - - cris-fsr_n20 - - cris-fsr_npp - - gome_metop-a - - gome_metop-b - - hirs4_metop-a - - hirs4_n19 - - iasi_metop-a - - iasi_metop-b - - mhs_metop-a - - mhs_metop-b - - mhs_metop-c - - mhs_n19 - - omi_aura - - ompsnp_npp - - ompstc8_npp - - rass_tv - #- saphir_meghat - - satwind - - sbuv2_n19 - - scatwind - - seviri_m08 - - seviri_m11 - - sfc - - sfcship - - sondes - - ssmis_f17 - - ssmis_f18 - - sst - - vadwind - #- windprof -EOF - $r2d2storepy --config $diagroot/r2d2_store_obs.yaml -fi - -if [ $r2d2store_bc = "YES" ]; then - cd $diagroot - nowpdy=${startdate::8}; nowcyc=${startdate:8} - firstdate=$(date -d "$nowpdy $nowcyc - 6 hour" +%Y-%m-%dT%H) - nowpdy=${enddate::8}; nowcyc=${enddate:8} - lastdate=$(date -d "$nowpdy $nowcyc - 6 hour" +%Y-%m-%dT%H) - cat > $diagroot/r2d2_store_satbias.yaml << EOF -start: ${firstdate}:00:00Z -end: ${lastdate}:00:00Z -step: PT6H -source_dir: $diagroot -source_file_fmt: '{source_dir}/satbias/{dump}.{year}{month}{day}/{hour}/atmos/{obs_type}_satbias.nc4' -type: bc -database: shared -provider: gsi -file_type: satbias -experiment: oper_$dump -obs_types: - - aircraft - - amsua_metop-a - - amsua_metop-b - - amsua_metop-c - - amsua_n15 - - amsua_n18 - - amsua_n19 - - atms_n20 - - atms_npp - - avhrr3_metop-a - - avhrr3_n18 - - cris-fsr_n20 - - cris-fsr_npp - - gome_metop-a - - gome_metop-b - - hirs4_metop-a - - hirs4_n19 - - iasi_metop-a - - iasi_metop-b - - mhs_metop-a - - mhs_metop-b - - mhs_metop-c - - mhs_n19 - - omi_aura - - ompsnp_npp - - ompstc8_npp - - rass_tv - #- saphir_meghat - - satwind - - sbuv2_n19 - - scatwind - - seviri_m08 - - seviri_m11 - - sfc - - sfcship - - sondes - - ssmis_f17 - - ssmis_f18 - - sst - - vadwind - #- windprof -EOF - $r2d2storepy --config $diagroot/r2d2_store_satbias.yaml - cat > $diagroot/r2d2_store_tlapse.yaml << EOF -start: ${firstdate}:00:00Z -end: ${lastdate}:00:00Z -step: PT6H -source_dir: $diagroot -source_file_fmt: '{source_dir}/satbias/{dump}.{year}{month}{day}/{hour}/atmos/{obs_type}_tlapmean.txt' -type: bc -database: shared -provider: gsi -file_type: tlapse -experiment: oper_$dump -obs_types: - - aircraft - - amsua_metop-a - - amsua_metop-b - - amsua_metop-c - - amsua_n15 - - amsua_n18 - - amsua_n19 - - atms_n20 - - atms_npp - - avhrr3_metop-a - - avhrr3_n18 - - cris-fsr_n20 - - cris-fsr_npp - - gome_metop-a - - gome_metop-b - - hirs4_metop-a - - hirs4_n19 - - iasi_metop-a - - iasi_metop-b - - mhs_metop-a - - mhs_metop-b - - mhs_metop-c - - mhs_n19 - - omi_aura - - ompsnp_npp - - ompstc8_npp - - rass_tv - #- saphir_meghat - - satwind - - sbuv2_n19 - - scatwind - - seviri_m08 - - seviri_m11 - - sfc - - sfcship - - sondes - - ssmis_f17 - - ssmis_f18 - - sst - - vadwind - #- windprof -EOF - $r2d2storepy --config $diagroot/r2d2_store_tlapse.yaml -fi diff --git a/ush/r2d2/r2d2_store.py b/ush/r2d2/r2d2_store.py deleted file mode 100755 index 98f982759..000000000 --- a/ush/r2d2/r2d2_store.py +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env python3 -import argparse -from solo.configuration import Configuration -import ufsda.r2d2 - -if __name__ == "__main__": - # get input config file - parser = argparse.ArgumentParser() - parser.add_argument('-c', '--config', type=str, help='Input YAML Configuration', required=True) - args = parser.parse_args() - config = Configuration(args.config) - ufsda.r2d2.store(config) diff --git a/ush/soca/examples/run_jjobs_hera.yaml b/ush/soca/examples/run_jjobs_hera.yaml index b11b0a332..b4bb5d448 100644 --- a/ush/soca/examples/run_jjobs_hera.yaml +++ b/ush/soca/examples/run_jjobs_hera.yaml @@ -37,8 +37,6 @@ setup_expt config: SOCA_OBS_LIST: '' COMIN_OBS: /scratch2/NCEPDEV/marineda/r2d2 SOCA_NINNER: 3 - R2D2_OBS_SRC: 'gdas_marine' - R2D2_OBS_DUMP: 's2s_v1' SABER_BLOCKS_YAML: '' NICAS_RESOL: 1 NICAS_GRID_SIZE: 15000 diff --git a/ush/soca/examples/run_jjobs_hera_025.yaml b/ush/soca/examples/run_jjobs_hera_025.yaml index 900167089..ba2552ef9 100644 --- a/ush/soca/examples/run_jjobs_hera_025.yaml +++ b/ush/soca/examples/run_jjobs_hera_025.yaml @@ -37,8 +37,6 @@ setup_expt config: SOCA_OBS_LIST: '' COMIN_OBS: /scratch2/NCEPDEV/marineda/r2d2 SOCA_NINNER: 1 - R2D2_OBS_SRC: 'gdas_marine' - R2D2_OBS_DUMP: 's2s_v1' SABER_BLOCKS_YAML: '' NICAS_RESOL: 1 NICAS_GRID_SIZE: 15000 diff --git a/ush/ufsda/__init__.py b/ush/ufsda/__init__.py index e74304171..eb34e17f4 100644 --- a/ush/ufsda/__init__.py +++ b/ush/ufsda/__init__.py @@ -1,7 +1,4 @@ from .disk_utils import mkdir, symlink from .ufs_yaml import gen_yaml, parse_config -import ufsda.archive -import ufsda.r2d2 -import ufsda.post import ufsda.yamltools from .misc_utils import isTrue, create_batch_job, submit_batch_job diff --git a/ush/ufsda/archive.py b/ush/ufsda/archive.py deleted file mode 100644 index 07bfc65ae..000000000 --- a/ush/ufsda/archive.py +++ /dev/null @@ -1,62 +0,0 @@ -from r2d2 import store -from solo.configuration import Configuration -from solo.nice_dict import NiceDict -import os -import shutil -import datetime as dt -import ufsda -from wxflow import YAMLFile - -__all__ = ['atm_diags'] - - -def atm_diags(config): - # fetch atm analysis obs - r2d2_config = { - 'start': config['prev_valid_time'], - 'end': config['prev_valid_time'], - 'step': config['atm_window_length'], - 'dump': 'gdas', # note: should move dump to config - 'experiment': config['experiment'], - } - r2d2_config = NiceDict(r2d2_config) - # get list of obs to process and their output files - obs_list_yaml = config['OBS_LIST'] - obs_list_config = YAMLFile(path=obs_list_yaml) - for ob in obs_list_config['observers']: - # first get obs - r2d2_config.pop('file_type', None) - r2d2_config['type'] = 'ob' - r2d2_config['provider'] = config['provider'] - r2d2_config['start'] = config['window_begin'] - r2d2_config['end'] = r2d2_config['start'] - input_file = ob['obs space']['obsdataout']['engine']['obsfile'] - r2d2_config['source_dir'] = config['OBS_DIR'] - r2d2_config['source_file_fmt'] = input_file.replace('.nc4', '_0000.nc4') - r2d2_config['obs_types'] = [ob['obs space']['name']] - ufsda.r2d2.store(r2d2_config) - # store bias files - if 'obs bias' in ob.keys(): - r2d2_config['type'] = 'bc' - r2d2_config['provider'] = 'gsi' - r2d2_config['start'] = config['valid_time'] - r2d2_config['end'] = config['valid_time'] - - # store satbias - r2d2_config['file_type'] = 'satbias' - target_file = ob['obs bias']['output file'] - r2d2_config['source_file_fmt'] = target_file - ufsda.r2d2.store(r2d2_config) - - # store satbias_cov # note: non-standard R2D2 added for cycling - r2d2_config['file_type'] = 'satbias_cov' - target_file = target_file.replace('satbias', 'satbias_cov') - r2d2_config['source_file_fmt'] = target_file - ufsda.r2d2.store(r2d2_config) - - # store tlapse - r2d2_config['file_type'] = 'tlapse' - target_file = target_file.replace('satbias', 'tlapse') - target_file = target_file.replace('nc4', 'txt') - r2d2_config['source_file_fmt'] = target_file - ufsda.r2d2.store(r2d2_config) diff --git a/ush/ufsda/post.py b/ush/ufsda/post.py deleted file mode 100644 index 9c45587c5..000000000 --- a/ush/ufsda/post.py +++ /dev/null @@ -1,73 +0,0 @@ -import os -import shutil -import glob -from solo.ioda import Ioda -from r2d2 import store - - -def merge_diags(config): - """ - Merges IODA diag files depending on input config dict - """ - # loop through list of observations - for ob in config['observations']: - obname = ob['obs space']['name'].lower() - outfile = ob['obs space']['obsdataout']['obsfile'] - # the above path is what 'FV3-JEDI' expects, need to modify it - outpath = outfile.split('/') - outpath[0] = 'analysis' - outpath = '/'.join(outpath) - outfile = os.path.join(config['COMOUT'], outpath) - outmatch = os.path.splitext(outfile)[0] + '_????.*' - diagfiles = glob.glob(outmatch) - if len(diagfiles) > 1: - concat_diags(diagfiles, outfile) - elif len(diagfiles) == 1: - shutil.copy(diagfiles[0], outfile) - else: - print(f"No output for {obname}. Skipping...") - - -def concat_diags(files, outfile): - """ - concat_diags(files, outfile) - for a list of files `files`, concatenate them into one `outfile` - """ - if (len(files)) > 0: - ioda = Ioda('diags') - ioda.concat_files(files, outfile) - else: - print('No files to concatenate!') - - -def archive_diags(config): - """ - Archives IODA diag files with R2D2 based on input config dict - """ - # loop through list of observations - for ob in config['observations']: - obname = ob['obs space']['name'].lower() - outfile = ob['obs space']['obsdataout']['obsfile'] - # the above path is what 'FV3-JEDI' expects, need to modify it - outpath = outfile.split('/') - outpath[0] = 'analysis' - outpath = '/'.join(outpath) - # below is assumed to be the final, concatenated diag file - diagfile = os.path.join(config['COMOUT'], outpath) - if os.path.exists(diagfile): - store( - type='diag', - experiment=config['experiment'], - date=config['window begin'], - model='gfs', - obs_type=obname.lower(), - source_file=diagfile, - database=config['r2d2_archive_db'], - ) - - -def cleanup(config): - """ - Performs final clean up to delete/move any remaining files - """ - print("No cleanup yet! PLACEHOLDER") diff --git a/ush/ufsda/r2d2.py b/ush/ufsda/r2d2.py deleted file mode 100644 index fc3b4c26e..000000000 --- a/ush/ufsda/r2d2.py +++ /dev/null @@ -1,115 +0,0 @@ -import r2d2 -import re -from solo.configuration import Configuration -from solo.date import date_sequence, Hour, DateIncrement -import yaml -import os - -possible_args = [ - 'provider', 'experiment', 'database', 'type', 'file_type', - 'resolution', 'model', 'user_date_format', 'fc_date_rendering', 'tile', -] - - -def setup(shared_root='', r2d2_config_yaml='r2d2_config.yaml'): - """ - setup(shared_root) - - prepares the R2D2 configuration yaml file and exports the R2D2_CONFIG - environement variable - - TODO (Guillaume): We need the flexibility to change all key values in the - below dictionary ... - """ - - # TODO: Should it be in a template instead? - r2d2_config = {'databases': {'archive': {'bucket': 'archive.jcsda', - 'cache_fetch': True, - 'class': 'S3DB'}, - 'local': {'cache_fetch': False, - 'class': 'LocalDB', - 'root': './r2d2-local/'}, - 'shared': {'cache_fetch': False, - 'class': 'LocalDB', - 'root': shared_root}}, - 'fetch_order': ['shared'], - 'store_order': ['local']} - - f = open(r2d2_config_yaml, 'w') - yaml.dump(r2d2_config, f, sort_keys=False, default_flow_style=False) - os.environ['R2D2_CONFIG'] = r2d2_config_yaml - - -def store(config): - inputs = {} - inputs['ignore_missing'] = True - for arg in config.keys(): - if arg in possible_args: - inputs[arg] = config[arg] - r2d2_type = config.type - times = date_sequence(config.start, config.end, config.step) - dump = config.get('dump', 'gdas') - source_dir = config['source_dir'] - source_file_fmt = config['source_file_fmt'] - obs_types = config.get('obs_types', None) - for time in times: - year = Hour(time).format('%Y') - month = Hour(time).format('%m') - day = Hour(time).format('%d') - hour = Hour(time).format('%H') - inputs['date'] = time - if r2d2_type in ['bc', 'ob', 'diag']: - if r2d2_type == 'ob': - inputs['date'] = time - inputs['time_window'] = config['step'] - for obs_type in obs_types: - inputs['source_file'] = eval(f"f'{source_file_fmt}'"), - inputs['obs_type'] = obs_type - r2d2.store(**inputs) - if r2d2_type in ['fc']: - inputs['model'] = config['model'] - if config['model'] == 'mom6_cice6_UFS': - inputs['step'] = config['forecast_steps'] - inputs['experiment'] = config['experiment'] - inputs['database'] = config['database'] - inputs['resolution'] = config['resolution'] - file_type = config['file_type'] - inputs['source_file'] = eval(f"f'{source_file_fmt}'"), - r2d2.store(**inputs) - else: - inputs['file_type'] = config.file_type_list - inputs['step'] = config['forecast_steps'] - inputs['source_file'] = eval(f"f'{source_file_fmt}'"), - r2d2.store(**inputs) - - -def fetch(config): - inputs = {} - inputs['ignore_missing'] = False - for arg in config.keys(): - if arg in possible_args: - inputs[arg] = config[arg] - r2d2_type = config.type - times = date_sequence(config.start, config.end, config.step) - dump = config.get('dump', 'gdas') - obs_types = config.get('obs_types', None) - target_dir = config['target_dir'] - target_file_fmt = config['target_file_fmt'] - for time in times: - year = Hour(time).format('%Y') - month = Hour(time).format('%m') - day = Hour(time).format('%d') - hour = Hour(time).format('%H') - inputs['date'] = time - if r2d2_type in ['bc', 'ob']: - if r2d2_type == 'ob': - inputs['time_window'] = config['step'] - for obs_type in obs_types: - inputs['target_file'] = eval(f"f'{target_file_fmt}'"), - inputs['obs_type'] = obs_type - r2d2.fetch(**inputs) - else: - inputs['file_type'] = config.file_type_list - inputs['step'] = config['forecast_steps'] - inputs['target_file'] = eval(f"f'{target_file_fmt}'"), - r2d2.fetch(**inputs) diff --git a/ush/ufsda/stage.py b/ush/ufsda/stage.py index 6698324fc..84c83a2ef 100644 --- a/ush/ufsda/stage.py +++ b/ush/ufsda/stage.py @@ -1,4 +1,3 @@ -from r2d2 import fetch from solo.basic_files import mkdir from solo.date import Hour, DateIncrement, date_sequence from solo.stage import Stage @@ -117,206 +116,6 @@ def soca_fix(config): os.path.join(config['stage_dir'], 'INPUT')) -def atm_background(config): - # stage FV3 backgrounds - r2d2_config = { - 'start': config['prev_valid_time'], - 'end': config['prev_valid_time'], - 'step': config['atm_window_length'], - 'forecast_steps': ['PT6H'], # 3DVar no FGAT for now - 'file_type_list': ['fv_core.res', 'fv_srf_wnd.res', 'fv_tracer.res', 'phy_data', 'sfc_data'], - 'target_dir': config.get('target_dir', config.get('BKG_DIR', './')), - 'target_file_fmt': '{target_dir}/$(valid_date).$(file_type).tile$(tile).nc', - 'type': 'fc', - 'model': 'gfs', - 'resolution': config['CASE'].lower(), - 'dump': 'gdas', - 'experiment': 'oper_gdas', # change this here and other places to be oper_{dump} - 'tile': [1, 2, 3, 4, 5, 6], - 'user_date_format': '%Y%m%d.%H%M%S', - 'fc_date_rendering': 'analysis', - } - r2d2_config = NiceDict(r2d2_config) - ufsda.r2d2.fetch(r2d2_config) - # get gfs metadata - r2d2_config['model'] = 'gfs_metadata' - r2d2_config['target_file_fmt'] = '{target_dir}/$(valid_date).$(file_type)' - r2d2_config['file_type_list'] = ['coupler.res', 'fv_core.res.nc'] - del r2d2_config['tile'] - ufsda.r2d2.fetch(r2d2_config) - - -def atm_obs(config, local_dict): - # fetch atm analysis obs - r2d2_config = { - 'start': config['prev_valid_time'], - 'end': config['prev_valid_time'], - 'step': config['ATM_WINDOW_LENGTH'], - 'dump': 'gdas', - 'experiment': 'oper_gdas', # change this here and other places to be oper_{dump} - 'target_dir': config.get('target_dir', os.path.join(config.get('DATA', './'), 'obs')), - } - r2d2_config = NiceDict(r2d2_config) - # get list of obs to process and their output files - obs_list_yaml = config['OBS_LIST'] - obs_list_config = Configuration(obs_list_yaml) - obs_list_config = ufsda.yamltools.iter_config(config, obs_list_config) - - for ob in obs_list_config['observers']: - ob_config = parse_j2yaml(ob, local_dict) - # first get obs - r2d2_config.pop('file_type', None) - r2d2_config['type'] = 'ob' - r2d2_config['provider'] = 'ncdiag' - r2d2_config['start'] = config['ATM_WINDOW_BEGIN'] - r2d2_config['end'] = r2d2_config['start'] - ob_basename = os.path.basename(ob_config['obs space']['obsdatain']['engine']['obsfile']) - target_file = os.path.join(os.environ['COMOUT'], ob_basename) - r2d2_config['target_file_fmt'] = target_file - r2d2_config['obs_types'] = [ob_config['obs space']['name']] - ufsda.r2d2.fetch(r2d2_config) - - -def bias_obs(config, local_dict): - # fetch bias files - r2d2_config = { - 'start': config['prev_valid_time'], - 'end': config['prev_valid_time'], - 'step': config['atm_window_length'], - 'dump': 'gdas', - 'experiment': 'oper_gdas', # change this here and other places to be oper_{dump} - 'target_dir': config.get('target_dir', config.get('COMIN_GES', './')), - } - r2d2_config = NiceDict(r2d2_config) - # get list of obs to process and their output files - obs_list_yaml = config['OBS_LIST'] - obs_list_config = Configuration(obs_list_yaml) - obs_list_config = ufsda.yamltools.iter_config(config, obs_list_config) - - for ob in obs_list_config['observers']: - ob_config = parse_j2yaml(ob, local_dict) - r2d2_config.pop('file_type', None) - r2d2_config['obs_types'] = [ob_config['obs space']['name']] - # get bias files if needed - if 'obs bias' in ob_config.keys(): - r2d2_config['type'] = 'bc' - r2d2_config['provider'] = 'gsi' - r2d2_config['start'] = config['prev_valid_time'] - r2d2_config['end'] = r2d2_config['start'] - - # fetch satbias - r2d2_config['file_type'] = 'satbias' - target_file = ob_config['obs bias']['input file'] - ob_basename = os.path.basename(target_file) - target_file = os.path.join(os.environ['COMOUT'], ob_basename) - r2d2_config['target_file_fmt'] = target_file - r2d2_config['experiment'] = config.get('experiment', 'oper_gdas') - ufsda.r2d2.fetch(r2d2_config) - - # fetch satbias_cov # note: non-standard R2D2 added for cycling - r2d2_config['file_type'] = 'satbias_cov' - target_file2 = target_file.replace('satbias', 'satbias_cov') - r2d2_config['target_file_fmt'] = target_file2 - r2d2_config['experiment'] = config.get('experiment', 'oper_gdas') - try: - ufsda.r2d2.fetch(r2d2_config) - except FileNotFoundError: - logging.error("Warning: satbias_cov file cannot be fetched from R2D2!") - # temp hack to copy satbias as satbias_cov - # if satbias_cov does not exists in R2D2 - if os.path.isfile(target_file) and not os.path.isfile(target_file2): - shutil.copy(target_file, target_file2) - - # fetch tlapse - r2d2_config['file_type'] = 'tlapse' - target_file = target_file.replace('satbias', 'tlapse') - target_file = target_file.replace('nc4', 'txt') - r2d2_config['target_file_fmt'] = target_file - r2d2_config['experiment'] = 'oper_gdas' - ufsda.r2d2.fetch(r2d2_config) - - -def gdas_single_cycle(config, local_dict): - # grab backgrounds first - r2d2_config = { - 'start': config['prev_valid_time'], - 'end': config['prev_valid_time'], - 'step': config['atm_window_length'], - 'forecast_steps': config['forecast_steps'], - 'file_type_list': ['fv_core.res', 'fv_srf_wnd.res', 'fv_tracer.res', 'phy_data', 'sfc_data'], - 'target_dir': config['BKG_DIR'], - 'target_file_fmt': '{target_dir}/$(valid_date).$(file_type).tile$(tile).nc', - 'type': 'fc', - 'model': 'gfs', - 'resolution': config['CASE'].lower(), - 'database': 'shared', - 'dump': 'gdas', - 'experiment': 'oper_gdas', # change this here and other places to be oper_{dump} - 'tile': [1, 2, 3, 4, 5, 6], - 'user_date_format': '%Y%m%d.%H%M%S', - 'fc_date_rendering': 'analysis', - } - r2d2_config = NiceDict(r2d2_config) - ufsda.r2d2.fetch(r2d2_config) - # get gfs metadata - r2d2_config['model'] = 'gfs_metadata' - r2d2_config['target_file_fmt'] = '{target_dir}/$(valid_date).$(file_type)' - r2d2_config['file_type_list'] = ['coupler.res', 'fv_core.res.nc'] - del r2d2_config['tile'] - ufsda.r2d2.fetch(r2d2_config) - - # remove things from dict and add/change for obs - delvars = ['forecast_steps', 'model', 'resolution', 'fc_date_rendering', 'user_date_format'] - for d in delvars: - del r2d2_config[d] - # get list of obs to process and their output files - obs_list_yaml = config['OBS_LIST'] - obs_list_config = Configuration(obs_list_yaml) - obs_list_config = ufsda.yamltools.iter_config(config, obs_list_config) - - for ob in obs_list_config['observers']: - ob_config = parse_yaml(ob, local_dict) - # first get obs - r2d2_config.pop('file_type', None) - r2d2_config['type'] = 'ob' - r2d2_config['provider'] = 'ncdiag' - r2d2_config['start'] = config['window_begin'] - r2d2_config['end'] = r2d2_config['start'] - target_file = ob_config['obs space']['obsdatain']['engine']['obsfile'] - r2d2_config['target_file_fmt'] = target_file - r2d2_config['obs_types'] = [ob_config['obs space']['name']] - ufsda.r2d2.fetch(r2d2_config) - # get bias files if needed - if 'obs bias' in ob_config.keys(): - r2d2_config['type'] = 'bc' - r2d2_config['provider'] = 'gsi' - r2d2_config['start'] = config['prev_valid_time'] - r2d2_config['end'] = r2d2_config['start'] - r2d2_config['file_type'] = 'satbias' - target_file = ob_config['obs bias']['input file'] - r2d2_config['target_file_fmt'] = target_file - ufsda.r2d2.fetch(r2d2_config) - r2d2_config['file_type'] = 'satbias_cov' - target_file2 = target_file.replace('satbias', 'satbias_cov') - r2d2_config['target_file_fmt'] = target_file2 - try: - ufsda.r2d2.fetch(r2d2_config) - except FileNotFoundError: - logging.error("Warning: satbias_cov file cannot be fetched from R2D2!") - # temp hack to copy satbias as satbias_cov - # if satbias_cov does not exists in R2D2 - if os.path.isfile(target_file) and not os.path.isfile(target_file2): - shutil.copy(target_file, target_file2) - r2d2_config['file_type'] = 'tlapse' - target_file = target_file.replace('satbias', 'tlapse') - target_file = target_file.replace('nc4', 'txt') - r2d2_config['target_file_fmt'] = target_file - ufsda.r2d2.fetch(r2d2_config) - # if hybvar, link ensemble members to DATA - if config['DOHYBVAR']: - background_ens(config) - - def background(config): """ Stage backgrounds and create analysis directory @@ -335,44 +134,6 @@ def background(config): mkdir(jedi_anl_dir) -def obs(config): - """ - Stage observations using R2D2 - based on input `config` dict - """ - for ob in config['observations']['observers']: - obname = ob['obs space']['name'].lower() - outfile = os.path.join(config['r2d2_obs_out'], - os.path.basename(ob['obs space']['obsdatain']['engine']['obsfile'])) - - # grab obs using R2D2 - window_begin = config['window begin'] - window_begin = parser.parse(window_begin, fuzzy=True) - window_end = window_begin + timedelta(hours=6) - steps = ['P1D', 'PT10M'] - for step in steps: - if step == 'P1D': - dates = date_sequence(window_begin.strftime('%Y%m%d'), window_end.strftime('%Y%m%d'), step) - if step == "PT10M": - dates = date_sequence(window_begin.strftime('%Y%m%d%H%M'), window_end.strftime('%Y%m%d%H%M'), step) - count = 0 - for date in dates: - fetch( - type='ob', - provider=config['r2d2_obs_src'], - experiment=config['r2d2_obs_dump'], - date=date, - obs_type=obname, - time_window=step, - target_file=outfile+'.'+str(count), - ignore_missing=True, - database=config['r2d2_obs_db'] - ) - count += 1 - # Concatenate ioda files - ufsda.soca_utils.concatenate_ioda(outfile) - - def fv3jedi(config): """ fv3jedi(config) From 6ad3b866fabab6a1354c25cfb659b737e85c01d4 Mon Sep 17 00:00:00 2001 From: Jiarui Dong Date: Thu, 11 Jan 2024 08:59:00 -0500 Subject: [PATCH 2/4] Add 'bound to include' to manipulate the DA window. (#846) Add `bound to include` to manipulate the DA window following the recent upgrades in JEDI on the DA window manipulation (PRs [ufo #3056](https://github.com/JCSDA-internal/ufo/pull/3056); [oops #2386](https://github.com/JCSDA-internal/oops/pull/2386) and [ioda #1121](https://github.com/JCSDA-internal/ioda/pull/1121)) for handling the loss of the observations due to the difference in NCEP dump convention and IODA convention. --- parm/land/letkfoi/letkfoi.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/parm/land/letkfoi/letkfoi.yaml b/parm/land/letkfoi/letkfoi.yaml index 55fc33ebf..da2540852 100644 --- a/parm/land/letkfoi/letkfoi.yaml +++ b/parm/land/letkfoi/letkfoi.yaml @@ -22,6 +22,7 @@ geometry: time window: begin: '{{ LAND_WINDOW_BEGIN | to_isotime }}' length: $(LAND_WINDOW_LENGTH) + bound to include: begin background: datetime: '{{ current_cycle | to_isotime }}' From 4e3e5868750115d3b0d8711e8060ae2275a7a985 Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Thu, 11 Jan 2024 11:55:53 -0500 Subject: [PATCH 3/4] Follow up of the "bye bye" R2D2 PR: More marine obs in the ctests (#847) - I moved all the untested obs `yaml` under `obs/old`, assuming that they probably are all wrong since the removal of R2D2 - I added (fake) AMSR2 icec cdl files - Same as above but for ADT, also skipping the concatenation post ioda processing and concatenating with the new ioda converters - I've made an effort to try to use the provider's file naming convention, but do check. --- .../{adt_all.yaml => adt_rads_all.yaml} | 6 +- .../obs/config/{ => bufr}/bufr_sfcships.yaml | 0 .../obs/config/{ => bufr}/bufr_sfcshipsu.yaml | 0 parm/soca/obs/config/obs_list.yaml | 1 - .../obs/config/{ => old}/adt_3a_egm2008.yaml | 0 .../obs/config/{ => old}/adt_3b_egm2008.yaml | 0 .../obs/config/{ => old}/adt_6a_egm2008.yaml | 0 .../obs/config/{ => old}/adt_c2_egm2008.yaml | 0 .../obs/config/{ => old}/adt_coperl4.yaml | 0 parm/soca/obs/config/{ => old}/adt_j2.yaml | 0 .../obs/config/{ => old}/adt_j2_egm2008.yaml | 0 parm/soca/obs/config/{ => old}/adt_j3.yaml | 0 .../obs/config/{ => old}/adt_j3_egm2008.yaml | 0 .../obs/config/{ => old}/adt_sa_egm2008.yaml | 0 parm/soca/obs/config/{ => old}/icec_emc.yaml | 0 .../obs/config/{ => old}/icec_nsidc_nh.yaml | 0 .../obs/config/{ => old}/icec_nsidc_sh.yaml | 0 .../{ => old}/icec_ssmis_f17_north.yaml | 0 .../{ => old}/icec_ssmis_f17_south.yaml | 0 .../{ => old}/icec_ssmis_f18_north.yaml | 0 .../{ => old}/icec_ssmis_f18_south.yaml | 0 parm/soca/obs/config/{ => old}/icefb_gdr.yaml | 0 parm/soca/obs/config/{ => old}/oc_aqua.yaml | 0 .../soca/obs/config/{ => old}/oc_dineof3.yaml | 0 parm/soca/obs/config/{ => old}/oc_noaa20.yaml | 0 parm/soca/obs/config/{ => old}/oc_snpp.yaml | 0 .../config/{ => old}/salt_bufr_dbuoyprof.yaml | 0 .../{ => old}/salt_bufr_mbuoybprof.yaml | 0 .../config/{ => old}/salt_profile_argo.yaml | 0 .../config/{ => old}/salt_profile_fnmoc.yaml | 0 .../{ => old}/salt_profile_fnmoc_godas.yaml | 0 .../config/{ => old}/salt_profile_glider.yaml | 0 .../config/{ => old}/salt_profile_tesac.yaml | 0 .../config/{ => old}/salt_profile_wod.yaml | 0 parm/soca/obs/config/{ => old}/sss_smap.yaml | 0 .../obs/config/{ => old}/sss_smap_jpl.yaml | 0 parm/soca/obs/config/{ => old}/sss_smos.yaml | 0 .../obs/config/{ => old}/sss_smos_esa.yaml | 0 .../obs/config/{ => old}/sss_trak_fnmoc.yaml | 0 .../obs/config/{ => old}/sst_amsr2_l3u.yaml | 0 .../obs/config/{ => old}/sst_drifter.yaml | 0 .../obs/config/{ => old}/sst_gmi_l3u.yaml | 0 .../{ => old}/sst_metopa_l3u_so025.yaml | 0 .../{ => old}/sst_metopc_l3u_so025.yaml | 0 .../{ => old}/sst_noaa18_l3u_so025.yaml | 0 .../obs/config/{ => old}/sst_noaa19_l3u.yaml | 0 .../{ => old}/sst_noaa19_l3u_so025.yaml | 0 parm/soca/obs/config/{ => old}/sst_ostia.yaml | 0 .../obs/config/{ => old}/sst_ship_fnmoc.yaml | 0 .../obs/config/{ => old}/sst_trak_fnmoc.yaml | 0 .../{ => old}/sst_viirs_n20_l3u_so025.yaml | 0 .../{ => old}/sst_viirs_npp_l3u_so025.yaml | 0 .../obs/config/{ => old}/sst_windsat_l3u.yaml | 0 .../config/{ => old}/temp_bufr_dbuoyprof.yaml | 0 .../{ => old}/temp_bufr_mbuoybprof.yaml | 0 .../config/{ => old}/temp_profile_argo.yaml | 0 .../config/{ => old}/temp_profile_bathy.yaml | 0 .../config/{ => old}/temp_profile_fnmoc.yaml | 0 .../{ => old}/temp_profile_fnmoc_godas.yaml | 0 .../config/{ => old}/temp_profile_glider.yaml | 0 .../config/{ => old}/temp_profile_tesac.yaml | 0 .../config/{ => old}/temp_profile_wod.yaml | 0 ...opb_l3u_so025.yaml => sst_metopb_l3u.yaml} | 6 +- parm/soca/obs/obs_list.yaml | 25 +- parm/soca/obs/obs_list_small.yaml | 9 - parm/soca/obsproc/obs_list_obsproc.yaml | 12 - parm/soca/obsproc/obsproc_config.yaml | 24 +- parm/soca/obsproc/obsproc_config_test.yaml | 35 -- scripts/exgdas_global_marine_analysis_prep.py | 7 +- test/soca/gw/prepdata.sh | 43 +-- test/soca/gw/run_jjobs.yaml.test | 6 +- test/soca/testdata/icec_amsr2_north_1.cdl | 208 ++++++++++++ test/soca/testdata/icec_amsr2_north_2.cdl | 208 ++++++++++++ test/soca/testdata/icec_amsr2_south_1.cdl | 306 ++++++++++++++++++ test/soca/testdata/icec_amsr2_south_2.cdl | 306 ++++++++++++++++++ test/soca/testdata/rads_adt_3a_2018105.cdl | 110 +++++++ test/soca/testdata/rads_adt_3b_2018105.cdl | 110 +++++++ ush/ufsda/soca_utils.py | 118 ------- ush/ufsda/stage.py | 1 - 79 files changed, 1305 insertions(+), 236 deletions(-) rename parm/soca/obs/config/{adt_all.yaml => adt_rads_all.yaml} (88%) rename parm/soca/obs/config/{ => bufr}/bufr_sfcships.yaml (100%) rename parm/soca/obs/config/{ => bufr}/bufr_sfcshipsu.yaml (100%) delete mode 120000 parm/soca/obs/config/obs_list.yaml rename parm/soca/obs/config/{ => old}/adt_3a_egm2008.yaml (100%) rename parm/soca/obs/config/{ => old}/adt_3b_egm2008.yaml (100%) rename parm/soca/obs/config/{ => old}/adt_6a_egm2008.yaml (100%) rename parm/soca/obs/config/{ => old}/adt_c2_egm2008.yaml (100%) rename parm/soca/obs/config/{ => old}/adt_coperl4.yaml (100%) rename parm/soca/obs/config/{ => old}/adt_j2.yaml (100%) rename parm/soca/obs/config/{ => old}/adt_j2_egm2008.yaml (100%) rename parm/soca/obs/config/{ => old}/adt_j3.yaml (100%) rename parm/soca/obs/config/{ => old}/adt_j3_egm2008.yaml (100%) rename parm/soca/obs/config/{ => old}/adt_sa_egm2008.yaml (100%) rename parm/soca/obs/config/{ => old}/icec_emc.yaml (100%) rename parm/soca/obs/config/{ => old}/icec_nsidc_nh.yaml (100%) rename parm/soca/obs/config/{ => old}/icec_nsidc_sh.yaml (100%) rename parm/soca/obs/config/{ => old}/icec_ssmis_f17_north.yaml (100%) rename parm/soca/obs/config/{ => old}/icec_ssmis_f17_south.yaml (100%) rename parm/soca/obs/config/{ => old}/icec_ssmis_f18_north.yaml (100%) rename parm/soca/obs/config/{ => old}/icec_ssmis_f18_south.yaml (100%) rename parm/soca/obs/config/{ => old}/icefb_gdr.yaml (100%) rename parm/soca/obs/config/{ => old}/oc_aqua.yaml (100%) rename parm/soca/obs/config/{ => old}/oc_dineof3.yaml (100%) rename parm/soca/obs/config/{ => old}/oc_noaa20.yaml (100%) rename parm/soca/obs/config/{ => old}/oc_snpp.yaml (100%) rename parm/soca/obs/config/{ => old}/salt_bufr_dbuoyprof.yaml (100%) rename parm/soca/obs/config/{ => old}/salt_bufr_mbuoybprof.yaml (100%) rename parm/soca/obs/config/{ => old}/salt_profile_argo.yaml (100%) rename parm/soca/obs/config/{ => old}/salt_profile_fnmoc.yaml (100%) rename parm/soca/obs/config/{ => old}/salt_profile_fnmoc_godas.yaml (100%) rename parm/soca/obs/config/{ => old}/salt_profile_glider.yaml (100%) rename parm/soca/obs/config/{ => old}/salt_profile_tesac.yaml (100%) rename parm/soca/obs/config/{ => old}/salt_profile_wod.yaml (100%) rename parm/soca/obs/config/{ => old}/sss_smap.yaml (100%) rename parm/soca/obs/config/{ => old}/sss_smap_jpl.yaml (100%) rename parm/soca/obs/config/{ => old}/sss_smos.yaml (100%) rename parm/soca/obs/config/{ => old}/sss_smos_esa.yaml (100%) rename parm/soca/obs/config/{ => old}/sss_trak_fnmoc.yaml (100%) rename parm/soca/obs/config/{ => old}/sst_amsr2_l3u.yaml (100%) rename parm/soca/obs/config/{ => old}/sst_drifter.yaml (100%) rename parm/soca/obs/config/{ => old}/sst_gmi_l3u.yaml (100%) rename parm/soca/obs/config/{ => old}/sst_metopa_l3u_so025.yaml (100%) rename parm/soca/obs/config/{ => old}/sst_metopc_l3u_so025.yaml (100%) rename parm/soca/obs/config/{ => old}/sst_noaa18_l3u_so025.yaml (100%) rename parm/soca/obs/config/{ => old}/sst_noaa19_l3u.yaml (100%) rename parm/soca/obs/config/{ => old}/sst_noaa19_l3u_so025.yaml (100%) rename parm/soca/obs/config/{ => old}/sst_ostia.yaml (100%) rename parm/soca/obs/config/{ => old}/sst_ship_fnmoc.yaml (100%) rename parm/soca/obs/config/{ => old}/sst_trak_fnmoc.yaml (100%) rename parm/soca/obs/config/{ => old}/sst_viirs_n20_l3u_so025.yaml (100%) rename parm/soca/obs/config/{ => old}/sst_viirs_npp_l3u_so025.yaml (100%) rename parm/soca/obs/config/{ => old}/sst_windsat_l3u.yaml (100%) rename parm/soca/obs/config/{ => old}/temp_bufr_dbuoyprof.yaml (100%) rename parm/soca/obs/config/{ => old}/temp_bufr_mbuoybprof.yaml (100%) rename parm/soca/obs/config/{ => old}/temp_profile_argo.yaml (100%) rename parm/soca/obs/config/{ => old}/temp_profile_bathy.yaml (100%) rename parm/soca/obs/config/{ => old}/temp_profile_fnmoc.yaml (100%) rename parm/soca/obs/config/{ => old}/temp_profile_fnmoc_godas.yaml (100%) rename parm/soca/obs/config/{ => old}/temp_profile_glider.yaml (100%) rename parm/soca/obs/config/{ => old}/temp_profile_tesac.yaml (100%) rename parm/soca/obs/config/{ => old}/temp_profile_wod.yaml (100%) rename parm/soca/obs/config/{sst_metopb_l3u_so025.yaml => sst_metopb_l3u.yaml} (84%) delete mode 100644 parm/soca/obs/obs_list_small.yaml delete mode 100644 parm/soca/obsproc/obs_list_obsproc.yaml delete mode 100644 parm/soca/obsproc/obsproc_config_test.yaml create mode 100644 test/soca/testdata/icec_amsr2_north_1.cdl create mode 100644 test/soca/testdata/icec_amsr2_north_2.cdl create mode 100644 test/soca/testdata/icec_amsr2_south_1.cdl create mode 100644 test/soca/testdata/icec_amsr2_south_2.cdl create mode 100644 test/soca/testdata/rads_adt_3a_2018105.cdl create mode 100644 test/soca/testdata/rads_adt_3b_2018105.cdl delete mode 100644 ush/ufsda/soca_utils.py diff --git a/parm/soca/obs/config/adt_all.yaml b/parm/soca/obs/config/adt_rads_all.yaml similarity index 88% rename from parm/soca/obs/config/adt_all.yaml rename to parm/soca/obs/config/adt_rads_all.yaml index b7c6697dc..fe41a50da 100644 --- a/parm/soca/obs/config/adt_all.yaml +++ b/parm/soca/obs/config/adt_rads_all.yaml @@ -1,13 +1,13 @@ obs space: - name: adt_all + name: adt_rads_all obsdatain: engine: type: H5File - obsfile: !ENV ${DATA}/obs/${OPREFIX}adt_all.${PDY}${cyc}.nc4 + obsfile: !ENV ${DATA}/obs/${OPREFIX}adt_rads_all.${PDY}${cyc}.nc4 obsdataout: engine: type: H5File - obsfile: !ENV ${DATA}/diags/adt_all.${PDY}${cyc}.nc4 + obsfile: !ENV ${DATA}/diags/adt_rads_all.${PDY}${cyc}.nc4 simulated variables: [absoluteDynamicTopography] io pool: max pool size: 1 diff --git a/parm/soca/obs/config/bufr_sfcships.yaml b/parm/soca/obs/config/bufr/bufr_sfcships.yaml similarity index 100% rename from parm/soca/obs/config/bufr_sfcships.yaml rename to parm/soca/obs/config/bufr/bufr_sfcships.yaml diff --git a/parm/soca/obs/config/bufr_sfcshipsu.yaml b/parm/soca/obs/config/bufr/bufr_sfcshipsu.yaml similarity index 100% rename from parm/soca/obs/config/bufr_sfcshipsu.yaml rename to parm/soca/obs/config/bufr/bufr_sfcshipsu.yaml diff --git a/parm/soca/obs/config/obs_list.yaml b/parm/soca/obs/config/obs_list.yaml deleted file mode 120000 index 372f2edc2..000000000 --- a/parm/soca/obs/config/obs_list.yaml +++ /dev/null @@ -1 +0,0 @@ -../obs_list.yaml \ No newline at end of file diff --git a/parm/soca/obs/config/adt_3a_egm2008.yaml b/parm/soca/obs/config/old/adt_3a_egm2008.yaml similarity index 100% rename from parm/soca/obs/config/adt_3a_egm2008.yaml rename to parm/soca/obs/config/old/adt_3a_egm2008.yaml diff --git a/parm/soca/obs/config/adt_3b_egm2008.yaml b/parm/soca/obs/config/old/adt_3b_egm2008.yaml similarity index 100% rename from parm/soca/obs/config/adt_3b_egm2008.yaml rename to parm/soca/obs/config/old/adt_3b_egm2008.yaml diff --git a/parm/soca/obs/config/adt_6a_egm2008.yaml b/parm/soca/obs/config/old/adt_6a_egm2008.yaml similarity index 100% rename from parm/soca/obs/config/adt_6a_egm2008.yaml rename to parm/soca/obs/config/old/adt_6a_egm2008.yaml diff --git a/parm/soca/obs/config/adt_c2_egm2008.yaml b/parm/soca/obs/config/old/adt_c2_egm2008.yaml similarity index 100% rename from parm/soca/obs/config/adt_c2_egm2008.yaml rename to parm/soca/obs/config/old/adt_c2_egm2008.yaml diff --git a/parm/soca/obs/config/adt_coperl4.yaml b/parm/soca/obs/config/old/adt_coperl4.yaml similarity index 100% rename from parm/soca/obs/config/adt_coperl4.yaml rename to parm/soca/obs/config/old/adt_coperl4.yaml diff --git a/parm/soca/obs/config/adt_j2.yaml b/parm/soca/obs/config/old/adt_j2.yaml similarity index 100% rename from parm/soca/obs/config/adt_j2.yaml rename to parm/soca/obs/config/old/adt_j2.yaml diff --git a/parm/soca/obs/config/adt_j2_egm2008.yaml b/parm/soca/obs/config/old/adt_j2_egm2008.yaml similarity index 100% rename from parm/soca/obs/config/adt_j2_egm2008.yaml rename to parm/soca/obs/config/old/adt_j2_egm2008.yaml diff --git a/parm/soca/obs/config/adt_j3.yaml b/parm/soca/obs/config/old/adt_j3.yaml similarity index 100% rename from parm/soca/obs/config/adt_j3.yaml rename to parm/soca/obs/config/old/adt_j3.yaml diff --git a/parm/soca/obs/config/adt_j3_egm2008.yaml b/parm/soca/obs/config/old/adt_j3_egm2008.yaml similarity index 100% rename from parm/soca/obs/config/adt_j3_egm2008.yaml rename to parm/soca/obs/config/old/adt_j3_egm2008.yaml diff --git a/parm/soca/obs/config/adt_sa_egm2008.yaml b/parm/soca/obs/config/old/adt_sa_egm2008.yaml similarity index 100% rename from parm/soca/obs/config/adt_sa_egm2008.yaml rename to parm/soca/obs/config/old/adt_sa_egm2008.yaml diff --git a/parm/soca/obs/config/icec_emc.yaml b/parm/soca/obs/config/old/icec_emc.yaml similarity index 100% rename from parm/soca/obs/config/icec_emc.yaml rename to parm/soca/obs/config/old/icec_emc.yaml diff --git a/parm/soca/obs/config/icec_nsidc_nh.yaml b/parm/soca/obs/config/old/icec_nsidc_nh.yaml similarity index 100% rename from parm/soca/obs/config/icec_nsidc_nh.yaml rename to parm/soca/obs/config/old/icec_nsidc_nh.yaml diff --git a/parm/soca/obs/config/icec_nsidc_sh.yaml b/parm/soca/obs/config/old/icec_nsidc_sh.yaml similarity index 100% rename from parm/soca/obs/config/icec_nsidc_sh.yaml rename to parm/soca/obs/config/old/icec_nsidc_sh.yaml diff --git a/parm/soca/obs/config/icec_ssmis_f17_north.yaml b/parm/soca/obs/config/old/icec_ssmis_f17_north.yaml similarity index 100% rename from parm/soca/obs/config/icec_ssmis_f17_north.yaml rename to parm/soca/obs/config/old/icec_ssmis_f17_north.yaml diff --git a/parm/soca/obs/config/icec_ssmis_f17_south.yaml b/parm/soca/obs/config/old/icec_ssmis_f17_south.yaml similarity index 100% rename from parm/soca/obs/config/icec_ssmis_f17_south.yaml rename to parm/soca/obs/config/old/icec_ssmis_f17_south.yaml diff --git a/parm/soca/obs/config/icec_ssmis_f18_north.yaml b/parm/soca/obs/config/old/icec_ssmis_f18_north.yaml similarity index 100% rename from parm/soca/obs/config/icec_ssmis_f18_north.yaml rename to parm/soca/obs/config/old/icec_ssmis_f18_north.yaml diff --git a/parm/soca/obs/config/icec_ssmis_f18_south.yaml b/parm/soca/obs/config/old/icec_ssmis_f18_south.yaml similarity index 100% rename from parm/soca/obs/config/icec_ssmis_f18_south.yaml rename to parm/soca/obs/config/old/icec_ssmis_f18_south.yaml diff --git a/parm/soca/obs/config/icefb_gdr.yaml b/parm/soca/obs/config/old/icefb_gdr.yaml similarity index 100% rename from parm/soca/obs/config/icefb_gdr.yaml rename to parm/soca/obs/config/old/icefb_gdr.yaml diff --git a/parm/soca/obs/config/oc_aqua.yaml b/parm/soca/obs/config/old/oc_aqua.yaml similarity index 100% rename from parm/soca/obs/config/oc_aqua.yaml rename to parm/soca/obs/config/old/oc_aqua.yaml diff --git a/parm/soca/obs/config/oc_dineof3.yaml b/parm/soca/obs/config/old/oc_dineof3.yaml similarity index 100% rename from parm/soca/obs/config/oc_dineof3.yaml rename to parm/soca/obs/config/old/oc_dineof3.yaml diff --git a/parm/soca/obs/config/oc_noaa20.yaml b/parm/soca/obs/config/old/oc_noaa20.yaml similarity index 100% rename from parm/soca/obs/config/oc_noaa20.yaml rename to parm/soca/obs/config/old/oc_noaa20.yaml diff --git a/parm/soca/obs/config/oc_snpp.yaml b/parm/soca/obs/config/old/oc_snpp.yaml similarity index 100% rename from parm/soca/obs/config/oc_snpp.yaml rename to parm/soca/obs/config/old/oc_snpp.yaml diff --git a/parm/soca/obs/config/salt_bufr_dbuoyprof.yaml b/parm/soca/obs/config/old/salt_bufr_dbuoyprof.yaml similarity index 100% rename from parm/soca/obs/config/salt_bufr_dbuoyprof.yaml rename to parm/soca/obs/config/old/salt_bufr_dbuoyprof.yaml diff --git a/parm/soca/obs/config/salt_bufr_mbuoybprof.yaml b/parm/soca/obs/config/old/salt_bufr_mbuoybprof.yaml similarity index 100% rename from parm/soca/obs/config/salt_bufr_mbuoybprof.yaml rename to parm/soca/obs/config/old/salt_bufr_mbuoybprof.yaml diff --git a/parm/soca/obs/config/salt_profile_argo.yaml b/parm/soca/obs/config/old/salt_profile_argo.yaml similarity index 100% rename from parm/soca/obs/config/salt_profile_argo.yaml rename to parm/soca/obs/config/old/salt_profile_argo.yaml diff --git a/parm/soca/obs/config/salt_profile_fnmoc.yaml b/parm/soca/obs/config/old/salt_profile_fnmoc.yaml similarity index 100% rename from parm/soca/obs/config/salt_profile_fnmoc.yaml rename to parm/soca/obs/config/old/salt_profile_fnmoc.yaml diff --git a/parm/soca/obs/config/salt_profile_fnmoc_godas.yaml b/parm/soca/obs/config/old/salt_profile_fnmoc_godas.yaml similarity index 100% rename from parm/soca/obs/config/salt_profile_fnmoc_godas.yaml rename to parm/soca/obs/config/old/salt_profile_fnmoc_godas.yaml diff --git a/parm/soca/obs/config/salt_profile_glider.yaml b/parm/soca/obs/config/old/salt_profile_glider.yaml similarity index 100% rename from parm/soca/obs/config/salt_profile_glider.yaml rename to parm/soca/obs/config/old/salt_profile_glider.yaml diff --git a/parm/soca/obs/config/salt_profile_tesac.yaml b/parm/soca/obs/config/old/salt_profile_tesac.yaml similarity index 100% rename from parm/soca/obs/config/salt_profile_tesac.yaml rename to parm/soca/obs/config/old/salt_profile_tesac.yaml diff --git a/parm/soca/obs/config/salt_profile_wod.yaml b/parm/soca/obs/config/old/salt_profile_wod.yaml similarity index 100% rename from parm/soca/obs/config/salt_profile_wod.yaml rename to parm/soca/obs/config/old/salt_profile_wod.yaml diff --git a/parm/soca/obs/config/sss_smap.yaml b/parm/soca/obs/config/old/sss_smap.yaml similarity index 100% rename from parm/soca/obs/config/sss_smap.yaml rename to parm/soca/obs/config/old/sss_smap.yaml diff --git a/parm/soca/obs/config/sss_smap_jpl.yaml b/parm/soca/obs/config/old/sss_smap_jpl.yaml similarity index 100% rename from parm/soca/obs/config/sss_smap_jpl.yaml rename to parm/soca/obs/config/old/sss_smap_jpl.yaml diff --git a/parm/soca/obs/config/sss_smos.yaml b/parm/soca/obs/config/old/sss_smos.yaml similarity index 100% rename from parm/soca/obs/config/sss_smos.yaml rename to parm/soca/obs/config/old/sss_smos.yaml diff --git a/parm/soca/obs/config/sss_smos_esa.yaml b/parm/soca/obs/config/old/sss_smos_esa.yaml similarity index 100% rename from parm/soca/obs/config/sss_smos_esa.yaml rename to parm/soca/obs/config/old/sss_smos_esa.yaml diff --git a/parm/soca/obs/config/sss_trak_fnmoc.yaml b/parm/soca/obs/config/old/sss_trak_fnmoc.yaml similarity index 100% rename from parm/soca/obs/config/sss_trak_fnmoc.yaml rename to parm/soca/obs/config/old/sss_trak_fnmoc.yaml diff --git a/parm/soca/obs/config/sst_amsr2_l3u.yaml b/parm/soca/obs/config/old/sst_amsr2_l3u.yaml similarity index 100% rename from parm/soca/obs/config/sst_amsr2_l3u.yaml rename to parm/soca/obs/config/old/sst_amsr2_l3u.yaml diff --git a/parm/soca/obs/config/sst_drifter.yaml b/parm/soca/obs/config/old/sst_drifter.yaml similarity index 100% rename from parm/soca/obs/config/sst_drifter.yaml rename to parm/soca/obs/config/old/sst_drifter.yaml diff --git a/parm/soca/obs/config/sst_gmi_l3u.yaml b/parm/soca/obs/config/old/sst_gmi_l3u.yaml similarity index 100% rename from parm/soca/obs/config/sst_gmi_l3u.yaml rename to parm/soca/obs/config/old/sst_gmi_l3u.yaml diff --git a/parm/soca/obs/config/sst_metopa_l3u_so025.yaml b/parm/soca/obs/config/old/sst_metopa_l3u_so025.yaml similarity index 100% rename from parm/soca/obs/config/sst_metopa_l3u_so025.yaml rename to parm/soca/obs/config/old/sst_metopa_l3u_so025.yaml diff --git a/parm/soca/obs/config/sst_metopc_l3u_so025.yaml b/parm/soca/obs/config/old/sst_metopc_l3u_so025.yaml similarity index 100% rename from parm/soca/obs/config/sst_metopc_l3u_so025.yaml rename to parm/soca/obs/config/old/sst_metopc_l3u_so025.yaml diff --git a/parm/soca/obs/config/sst_noaa18_l3u_so025.yaml b/parm/soca/obs/config/old/sst_noaa18_l3u_so025.yaml similarity index 100% rename from parm/soca/obs/config/sst_noaa18_l3u_so025.yaml rename to parm/soca/obs/config/old/sst_noaa18_l3u_so025.yaml diff --git a/parm/soca/obs/config/sst_noaa19_l3u.yaml b/parm/soca/obs/config/old/sst_noaa19_l3u.yaml similarity index 100% rename from parm/soca/obs/config/sst_noaa19_l3u.yaml rename to parm/soca/obs/config/old/sst_noaa19_l3u.yaml diff --git a/parm/soca/obs/config/sst_noaa19_l3u_so025.yaml b/parm/soca/obs/config/old/sst_noaa19_l3u_so025.yaml similarity index 100% rename from parm/soca/obs/config/sst_noaa19_l3u_so025.yaml rename to parm/soca/obs/config/old/sst_noaa19_l3u_so025.yaml diff --git a/parm/soca/obs/config/sst_ostia.yaml b/parm/soca/obs/config/old/sst_ostia.yaml similarity index 100% rename from parm/soca/obs/config/sst_ostia.yaml rename to parm/soca/obs/config/old/sst_ostia.yaml diff --git a/parm/soca/obs/config/sst_ship_fnmoc.yaml b/parm/soca/obs/config/old/sst_ship_fnmoc.yaml similarity index 100% rename from parm/soca/obs/config/sst_ship_fnmoc.yaml rename to parm/soca/obs/config/old/sst_ship_fnmoc.yaml diff --git a/parm/soca/obs/config/sst_trak_fnmoc.yaml b/parm/soca/obs/config/old/sst_trak_fnmoc.yaml similarity index 100% rename from parm/soca/obs/config/sst_trak_fnmoc.yaml rename to parm/soca/obs/config/old/sst_trak_fnmoc.yaml diff --git a/parm/soca/obs/config/sst_viirs_n20_l3u_so025.yaml b/parm/soca/obs/config/old/sst_viirs_n20_l3u_so025.yaml similarity index 100% rename from parm/soca/obs/config/sst_viirs_n20_l3u_so025.yaml rename to parm/soca/obs/config/old/sst_viirs_n20_l3u_so025.yaml diff --git a/parm/soca/obs/config/sst_viirs_npp_l3u_so025.yaml b/parm/soca/obs/config/old/sst_viirs_npp_l3u_so025.yaml similarity index 100% rename from parm/soca/obs/config/sst_viirs_npp_l3u_so025.yaml rename to parm/soca/obs/config/old/sst_viirs_npp_l3u_so025.yaml diff --git a/parm/soca/obs/config/sst_windsat_l3u.yaml b/parm/soca/obs/config/old/sst_windsat_l3u.yaml similarity index 100% rename from parm/soca/obs/config/sst_windsat_l3u.yaml rename to parm/soca/obs/config/old/sst_windsat_l3u.yaml diff --git a/parm/soca/obs/config/temp_bufr_dbuoyprof.yaml b/parm/soca/obs/config/old/temp_bufr_dbuoyprof.yaml similarity index 100% rename from parm/soca/obs/config/temp_bufr_dbuoyprof.yaml rename to parm/soca/obs/config/old/temp_bufr_dbuoyprof.yaml diff --git a/parm/soca/obs/config/temp_bufr_mbuoybprof.yaml b/parm/soca/obs/config/old/temp_bufr_mbuoybprof.yaml similarity index 100% rename from parm/soca/obs/config/temp_bufr_mbuoybprof.yaml rename to parm/soca/obs/config/old/temp_bufr_mbuoybprof.yaml diff --git a/parm/soca/obs/config/temp_profile_argo.yaml b/parm/soca/obs/config/old/temp_profile_argo.yaml similarity index 100% rename from parm/soca/obs/config/temp_profile_argo.yaml rename to parm/soca/obs/config/old/temp_profile_argo.yaml diff --git a/parm/soca/obs/config/temp_profile_bathy.yaml b/parm/soca/obs/config/old/temp_profile_bathy.yaml similarity index 100% rename from parm/soca/obs/config/temp_profile_bathy.yaml rename to parm/soca/obs/config/old/temp_profile_bathy.yaml diff --git a/parm/soca/obs/config/temp_profile_fnmoc.yaml b/parm/soca/obs/config/old/temp_profile_fnmoc.yaml similarity index 100% rename from parm/soca/obs/config/temp_profile_fnmoc.yaml rename to parm/soca/obs/config/old/temp_profile_fnmoc.yaml diff --git a/parm/soca/obs/config/temp_profile_fnmoc_godas.yaml b/parm/soca/obs/config/old/temp_profile_fnmoc_godas.yaml similarity index 100% rename from parm/soca/obs/config/temp_profile_fnmoc_godas.yaml rename to parm/soca/obs/config/old/temp_profile_fnmoc_godas.yaml diff --git a/parm/soca/obs/config/temp_profile_glider.yaml b/parm/soca/obs/config/old/temp_profile_glider.yaml similarity index 100% rename from parm/soca/obs/config/temp_profile_glider.yaml rename to parm/soca/obs/config/old/temp_profile_glider.yaml diff --git a/parm/soca/obs/config/temp_profile_tesac.yaml b/parm/soca/obs/config/old/temp_profile_tesac.yaml similarity index 100% rename from parm/soca/obs/config/temp_profile_tesac.yaml rename to parm/soca/obs/config/old/temp_profile_tesac.yaml diff --git a/parm/soca/obs/config/temp_profile_wod.yaml b/parm/soca/obs/config/old/temp_profile_wod.yaml similarity index 100% rename from parm/soca/obs/config/temp_profile_wod.yaml rename to parm/soca/obs/config/old/temp_profile_wod.yaml diff --git a/parm/soca/obs/config/sst_metopb_l3u_so025.yaml b/parm/soca/obs/config/sst_metopb_l3u.yaml similarity index 84% rename from parm/soca/obs/config/sst_metopb_l3u_so025.yaml rename to parm/soca/obs/config/sst_metopb_l3u.yaml index 8865929c1..6b2fc9ac5 100644 --- a/parm/soca/obs/config/sst_metopb_l3u_so025.yaml +++ b/parm/soca/obs/config/sst_metopb_l3u.yaml @@ -1,13 +1,13 @@ obs space: - name: sst_metopb_l3u_so025 + name: sst_metopb_l3u obsdatain: engine: type: H5File - obsfile: !ENV ${DATA}/obs/${OPREFIX}sst_metopb_l3u_so025.${PDY}${cyc}.nc4 + obsfile: !ENV ${DATA}/obs/${OPREFIX}sst_metopb_l3u.${PDY}${cyc}.nc4 obsdataout: engine: type: H5File - obsfile: !ENV ${DATA}/diags/sst_metopb_l3u_so025.${PDY}${cyc}.nc4 + obsfile: !ENV ${DATA}/diags/sst_metopb_l3u.${PDY}${cyc}.nc4 simulated variables: [seaSurfaceTemperature] io pool: max pool size: 1 diff --git a/parm/soca/obs/obs_list.yaml b/parm/soca/obs/obs_list.yaml index 89b2659a1..811872f66 100644 --- a/parm/soca/obs/obs_list.yaml +++ b/parm/soca/obs/obs_list.yaml @@ -1,22 +1,5 @@ observers: -- !INC ${OBS_YAML_DIR}/adt_all.yaml -- !INC ${OBS_YAML_DIR}/adt_j3_egm2008.yaml -- !INC ${OBS_YAML_DIR}/adt_3a_egm2008.yaml -- !INC ${OBS_YAML_DIR}/adt_3b_egm2008.yaml -- !INC ${OBS_YAML_DIR}/adt_c2_egm2008.yaml -- !INC ${OBS_YAML_DIR}/adt_sa_egm2008.yaml -- !INC ${OBS_YAML_DIR}/sst_viirs_npp_l3u_so025.yaml -#- !INC ${OBS_YAML_DIR}/sst_metopa_l3u_so025.yaml -#- !INC ${OBS_YAML_DIR}/sst_metopb_l3u_so025.yaml -#- !INC ${OBS_YAML_DIR}/sst_metopc_l3u_so025.yaml -- !INC ${OBS_YAML_DIR}/icec_ssmis_f17_north.yaml -#- !INC ${OBS_YAML_DIR}/icec_ssmis_f17_south.yaml -#- !INC ${OBS_YAML_DIR}/icec_ssmis_f18_north.yaml -- !INC ${OBS_YAML_DIR}/icec_ssmis_f18_south.yaml -#- !INC ${OBS_YAML_DIR}/temp_profile_argo.yaml -#- !INC ${OBS_YAML_DIR}/temp_profile_bathy.yaml -#- !INC ${OBS_YAML_DIR}/temp_profile_glider.yaml -#- !INC ${OBS_YAML_DIR}/temp_profile_tesac.yaml -#- !INC ${OBS_YAML_DIR}/salt_profile_argo.yaml -#- !INC ${OBS_YAML_DIR}/salt_profile_glider.yaml -#- !INC ${OBS_YAML_DIR}/salt_profile_tesac.yaml \ No newline at end of file +- !INC ${OBS_YAML_DIR}/adt_rads_all.yaml +- !INC ${OBS_YAML_DIR}/sst_metopb_l3u.yaml +- !INC ${OBS_YAML_DIR}/icec_amsr2_north.yaml +- !INC ${OBS_YAML_DIR}/icec_amsr2_south.yaml diff --git a/parm/soca/obs/obs_list_small.yaml b/parm/soca/obs/obs_list_small.yaml deleted file mode 100644 index 4ff437200..000000000 --- a/parm/soca/obs/obs_list_small.yaml +++ /dev/null @@ -1,9 +0,0 @@ -observers: -- !INC ${OBS_YAML_DIR}/adt_all.yaml -- !INC ${OBS_YAML_DIR}/adt_j3.yaml -- !INC ${OBS_YAML_DIR}/adt_j2.yaml -#- !INC ${OBS_YAML_DIR}/salt_profile_fnmoc.yaml -- !INC ${OBS_YAML_DIR}/sss_smap.yaml -- !INC ${OBS_YAML_DIR}/sst_metopb_l3u_so025.yaml -- !INC ${OBS_YAML_DIR}/sst_noaa19_l3u.yaml -- !INC ${OBS_YAML_DIR}/icec_emc.yaml diff --git a/parm/soca/obsproc/obs_list_obsproc.yaml b/parm/soca/obsproc/obs_list_obsproc.yaml deleted file mode 100644 index 7a884f9ea..000000000 --- a/parm/soca/obsproc/obs_list_obsproc.yaml +++ /dev/null @@ -1,12 +0,0 @@ -observers: -- !INC ${OBS_YAML_DIR}/adt_all.yaml -- !INC ${OBS_YAML_DIR}/adt_j3.yaml -- !INC ${OBS_YAML_DIR}/adt_j2.yaml -#- !INC ${OBS_YAML_DIR}/salt_profile_fnmoc.yaml -- !INC ${OBS_YAML_DIR}/sss_smap.yaml -- !INC ${OBS_YAML_DIR}/sss_smos.yaml -#- !INC ${OBS_YAML_DIR}/sst_noaa19_l3u.yaml -- !INC ${OBS_YAML_DIR}/sst_metopb_l3u_so025.yaml -- !INC ${OBS_YAML_DIR}/icec_emc.yaml -- !INC ${OBS_YAML_DIR}/adt_3b_egm2008.yaml -- !INC ${OBS_YAML_DIR}/icec_amsr2_north.yaml diff --git a/parm/soca/obsproc/obsproc_config.yaml b/parm/soca/obsproc/obsproc_config.yaml index 60cb0df66..5102ac9a3 100644 --- a/parm/soca/obsproc/obsproc_config.yaml +++ b/parm/soca/obsproc/obsproc_config.yaml @@ -12,14 +12,30 @@ observations: output file: sss_smos.ioda.nc obsproc regex: SM_OPER_MIR_OSUDP2_????????T??????_????????T??????_700_001_1.nc - obs space: - name: adt_3b_egm2008 + name: adt_rads_all obsproc subdir: ADT - obsproc regex: rads_adt_3b_???????.nc + obsproc regex: rads_adt_??_???????.nc provider: RADS - output file: adt_all.nc4 + output file: adt_rads_all.nc4 - obs space: name: icec_amsr2_north provider: AMSR2 obsproc subdir: icec - output file: icec_amsr2_north_1.ioda.nc + output file: icec_amsr2_north.ioda.nc obsproc regex: AMSR2-SEAICE-NH_v2r2_GW1_s???????????????_e???????????????_c???????????????.nc +- obs space: + name: icec_amsr2_south + provider: AMSR2 + obsproc subdir: icec + output file: icec_amsr2_south.ioda.nc + obsproc regex: AMSR2-SEAICE-SH_v2r2_GW1_s???????????????_e???????????????_c???????????????.nc +- obs space: + name: sst_metopb_l3u + provider: GHRSST + obsproc subdir: 'sst' + output file: sst_metopb_l3u.ioda.nc + obsproc regex: '??????????????-STAR-L3U_GHRSST-SSTsubskin-AVHRRF_MB-ACSPO_V2.80-v02.0-fv01.0.nc' + bounds: + units: C + min: -3.0 + max: 50.0 diff --git a/parm/soca/obsproc/obsproc_config_test.yaml b/parm/soca/obsproc/obsproc_config_test.yaml deleted file mode 100644 index 288aee4f4..000000000 --- a/parm/soca/obsproc/obsproc_config_test.yaml +++ /dev/null @@ -1,35 +0,0 @@ -observations: -- obs space: - name: sss_smap - obsproc subdir: 'SSS' - obsproc regex: sss_smap_?.nc4 - provider: SMAP - output file: sss_smap.ioda.nc -- obs space: - name: sss_smos - provider: SMOS - obsproc subdir: 'SSS' - output file: sss_smos.ioda.nc - obsproc regex: sss_smos_?.nc4 -- obs space: - name: adt_3b_egm2008 - obsproc subdir: 'ADT' - obsproc regex: rads_adt_??_???????.nc4 - provider: RADS - output file: adt_all.nc4 -- obs space: - name: icec_amsr2_north - provider: AMSR2 - obsproc subdir: 'ice' - output file: icec_amsr2_north.ioda.nc - obsproc regex: icec_amsr2_north_?.nc4 -- obs space: - name: sst_metopb_l3u_so025 - provider: GHRSST - obsproc subdir: 'sst' - output file: sst_metopb_l3u_so025.ioda.nc - obsproc regex: '??????????????-STAR-L3U_GHRSST-SSTsubskin-AVHRRF_MB-ACSPO_V2.80-v02.0-fv01.0.nc' - bounds: - units: C - min: -3.0 - max: 50.0 diff --git a/scripts/exgdas_global_marine_analysis_prep.py b/scripts/exgdas_global_marine_analysis_prep.py index 4af8d0189..d2cda4b1b 100755 --- a/scripts/exgdas_global_marine_analysis_prep.py +++ b/scripts/exgdas_global_marine_analysis_prep.py @@ -225,7 +225,7 @@ def find_clim_ens(input_date): logging.info(f"---------------- Setup runtime environement") -comin_obs = os.getenv('COMIN_OBS') # R2D2 DB for now +comin_obs = os.getenv('COMIN_OBS') anl_dir = os.getenv('DATA') staticsoca_dir = os.getenv('SOCA_INPUT_FIX_DIR') if os.getenv('DOHYBVAR') == "YES": @@ -268,11 +268,6 @@ def find_clim_ens(input_date): stage_cfg = Template.substitute_structure(stage_cfg, TemplateConstants.DOUBLE_CURLY_BRACES, envconfig.get) stage_cfg = Template.substitute_structure(stage_cfg, TemplateConstants.DOLLAR_PARENTHESES, envconfig.get) -# concatenate altimeters into one obs space -# TODO (SAMG)temporary, move this into the obs procecing eventually -adt_obs = f"{os.getenv('COM_OBS')}/{RUN}.t{cyc}z.adt" -ufsda.soca_utils.concatenate_ioda(adt_obs, wildcard="*.nc4", output_suffix=f"_all.{PDY}{cyc}.nc4", clean=True) - # get the list of observations obs_files = [] for ob in stage_cfg['observations']['observers']: diff --git a/test/soca/gw/prepdata.sh b/test/soca/gw/prepdata.sh index 88cde39bf..74f86b3bc 100755 --- a/test/soca/gw/prepdata.sh +++ b/test/soca/gw/prepdata.sh @@ -16,23 +16,26 @@ cdl2nc4() { project_source_dir=$1 -#cdl2nc4 ADT/rads_adt_3a_2021181.nc4 ${project_source_dir}/testdata/rads_adt_3a_2021181.cdl -#cdl2nc4 ADT/rads_adt_3b_2021181.nc4 ${project_source_dir}/testdata/rads_adt_3b_2021181.cdl -#cdl2nc4 icec/icec_amsr2_north_1.nc4 ${project_source_dir}/testdata/icec_amsr2_north_1.cdl -#cdl2nc4 icec/icec_amsr2_north_2.nc4 ${project_source_dir}/testdata/icec_amsr2_north_2.cdl -#cdl2nc4 icec/icec_amsr2_south_1.nc4 ${project_source_dir}/testdata/icec_amsr2_south_1.cdl -#cdl2nc4 icec/icec_amsr2_south_2.nc4 ${project_source_dir}/testdata/icec_amsr2_south_2.cdl -#cdl2nc4 SSS/sss_smap_1.nc4 ${project_source_dir}/testdata/sss_smap_1.cdl -#cdl2nc4 SSS/sss_smap_2.nc4 ${project_source_dir}/testdata/sss_smap_2.cdl -#cdl2nc4 SSS/sss_smos_1.nc4 ${project_source_dir}/testdata/sss_smos_1.cdl -#cdl2nc4 SSS/sss_smos_2.nc4 ${project_source_dir}/testdata/sss_smos_2.cdl -#cdl2nc4 sst/ghrsst_sst_mb_202107010000.nc4 ${project_source_dir}/testdata/ghrsst_sst_mb_202107010000.cdl -#cdl2nc4 sst/ghrsst_sst_mb_202107010100.nc4 ${project_source_dir}/testdata/ghrsst_sst_mb_202107010100.cdl -#cdl2nc4 sst/viirs_aod_1.nc4 ${project_source_dir}/testdata/viirs_aod_1.cdl -#cdl2nc4 sst/viirs_aod_2.nc4 ${project_source_dir}/testdata/viirs_aod_2.cdl - -cdl2nc4 sst/20180415114000-STAR-L3U_GHRSST-SSTsubskin-AVHRRF_MB-ACSPO_V2.80-v02.0-fv01.0.nc ${project_source_dir}/test/soca/testdata/20180415114000-STAR-L3U_GHRSST-SSTsubskin-AVHRRF_MB-ACSPO_V2.80-v02.0-fv01.0.cdl -cdl2nc4 sst/20180415132000-STAR-L3U_GHRSST-SSTsubskin-AVHRRF_MB-ACSPO_V2.80-v02.0-fv01.0.nc ${project_source_dir}/test/soca/testdata/20180415132000-STAR-L3U_GHRSST-SSTsubskin-AVHRRF_MB-ACSPO_V2.80-v02.0-fv01.0.cdl -cdl2nc4 sst/20180415095000-STAR-L3U_GHRSST-SSTsubskin-AVHRRF_MB-ACSPO_V2.80-v02.0-fv01.0.nc ${project_source_dir}/test/soca/testdata/20180415095000-STAR-L3U_GHRSST-SSTsubskin-AVHRRF_MB-ACSPO_V2.80-v02.0-fv01.0.cdl - - +cdldir=${project_source_dir}/test/soca/testdata + +# GHRSST +cdl2nc4 sst/20180415114000-STAR-L3U_GHRSST-SSTsubskin-AVHRRF_MB-ACSPO_V2.80-v02.0-fv01.0.nc \ + ${cdldir}/20180415114000-STAR-L3U_GHRSST-SSTsubskin-AVHRRF_MB-ACSPO_V2.80-v02.0-fv01.0.cdl +cdl2nc4 sst/20180415132000-STAR-L3U_GHRSST-SSTsubskin-AVHRRF_MB-ACSPO_V2.80-v02.0-fv01.0.nc \ + ${cdldir}/20180415132000-STAR-L3U_GHRSST-SSTsubskin-AVHRRF_MB-ACSPO_V2.80-v02.0-fv01.0.cdl +cdl2nc4 sst/20180415095000-STAR-L3U_GHRSST-SSTsubskin-AVHRRF_MB-ACSPO_V2.80-v02.0-fv01.0.nc \ + ${cdldir}/20180415095000-STAR-L3U_GHRSST-SSTsubskin-AVHRRF_MB-ACSPO_V2.80-v02.0-fv01.0.cdl + +# Fake AMSR2 icec +cdl2nc4 icec/AMSR2-SEAICE-NH_v2r2_GW1_s201804150835180_e201804151014170_c201804151052280.nc \ + ${project_source_dir}/test/soca/testdata/icec_amsr2_north_1.cdl +cdl2nc4 icec/AMSR2-SEAICE-NH_v2r2_GW1_s201804151014190_e201804151150170_c201804151309570.nc \ + ${project_source_dir}/test/soca/testdata/icec_amsr2_north_2.cdl +cdl2nc4 icec/AMSR2-SEAICE-SH_v2r2_GW1_s201804150835180_e201804151014170_c201804151052280.nc \ + ${project_source_dir}/test/soca/testdata/icec_amsr2_south_1.cdl +cdl2nc4 icec/AMSR2-SEAICE-SH_v2r2_GW1_s201804151014190_e201804151150170_c201804151309570.nc \ + ${project_source_dir}/test/soca/testdata/icec_amsr2_south_2.cdl + +# Fake RADS ADT +cdl2nc4 ADT/rads_adt_3a_2018105.nc ${project_source_dir}/test/soca/testdata/rads_adt_3a_2018105.cdl +cdl2nc4 ADT/rads_adt_3b_2018105.nc ${project_source_dir}/test/soca/testdata/rads_adt_3b_2018105.cdl diff --git a/test/soca/gw/run_jjobs.yaml.test b/test/soca/gw/run_jjobs.yaml.test index ee42ad7f6..a185cee2b 100644 --- a/test/soca/gw/run_jjobs.yaml.test +++ b/test/soca/gw/run_jjobs.yaml.test @@ -48,14 +48,14 @@ setup_expt config: ocnanal: SOCA_INPUT_FIX_DIR: @HOMEgfs@/sorc/gdas.cd/build/soca_static CASE_ANL: C48 - SOCA_OBS_LIST: @HOMEgfs@/sorc/gdas.cd/parm/soca/obs/obs_list_small.yaml + SOCA_OBS_LIST: @HOMEgfs@/sorc/gdas.cd/parm/soca/obs/obs_list.yaml SOCA_NINNER: 1 SABER_BLOCKS_YAML: @HOMEgfs@/sorc/gdas.cd/parm/soca/berror/saber_blocks.yaml NICAS_RESOL: 1 NICAS_GRID_SIZE: 150 prepoceanobs: - SOCA_OBS_LIST: @HOMEgfs@/sorc/gdas.cd/parm/soca/obsproc/obs_list_obsproc.yaml - OBSPROC_CONFIG: @HOMEgfs@/sorc/gdas.cd/parm/soca/obsproc/obsproc_config_test.yaml + SOCA_OBS_LIST: @HOMEgfs@/sorc/gdas.cd/parm/soca/obs/obs_list.yaml + OBSPROC_CONFIG: @HOMEgfs@/sorc/gdas.cd/parm/soca/obsproc/obsproc_config.yaml DMPDIR: @HOMEgfs@/sorc/gdas.cd/build/test/soca/gw/obsproc job options: diff --git a/test/soca/testdata/icec_amsr2_north_1.cdl b/test/soca/testdata/icec_amsr2_north_1.cdl new file mode 100644 index 000000000..9470a6bdb --- /dev/null +++ b/test/soca/testdata/icec_amsr2_north_1.cdl @@ -0,0 +1,208 @@ +netcdf output { +dimensions: + Number_of_X_Dimension = 3 ; + Number_of_Y_Dimension = 4 ; + QM = 1 ; + Time_Dimension = 6 ; +variables: + short Across_X_Dimension(Number_of_X_Dimension) ; + Across_X_Dimension:long_name = " Across X Dimension" ; + Across_X_Dimension:coordinates = "" ; + Across_X_Dimension:units = "1" ; + Across_X_Dimension:_FillValue = -9999s ; + short Along_Y_Dimension(Number_of_Y_Dimension) ; + Along_Y_Dimension:long_name = " Along Y Dimension" ; + Along_Y_Dimension:coordinates = "" ; + Along_Y_Dimension:units = "1" ; + Along_Y_Dimension:_FillValue = -9999s ; + int Bootstrap_Ice_Concentration(Number_of_Y_Dimension, Number_of_X_Dimension) ; + Bootstrap_Ice_Concentration:long_name = "Bootstrap Ice Concentration (Fraction) - secondary product" ; + Bootstrap_Ice_Concentration:units = "1" ; + Bootstrap_Ice_Concentration:coordinates = "Latitude Longitude" ; + Bootstrap_Ice_Concentration:_FillValue = -9999 ; + short Flags(Number_of_Y_Dimension, Number_of_X_Dimension) ; + Flags:long_name = "Quality Flags" ; + Flags:units = "1" ; + Flags:coordinate = "Latitude Longitude" ; + Flags:_FillValue = -9999s ; + Flags:comment = "0: good, 4: SST missing, 8: Weather, 16: spillover, 32: filled, 64: NT2 Missing" ; + int Latency(Number_of_Y_Dimension, Number_of_X_Dimension) ; + Latency:long_name = "Time in seconds from most current time" ; + Latency:units = "seconds" ; + Latency:coordinate = "Latitude Longitude" ; + Latency:_FillValue = -9999 ; + float Latitude(Number_of_Y_Dimension, Number_of_X_Dimension) ; + Latitude:long_name = "Latitude for EASE-2 grid" ; + Latitude:units = "degrees_north" ; + Latitude:valid_range = -90.f, 90.f ; + Latitude:_FillValue = -9999.f ; + Latitude:standard_name = "latitude" ; + float Longitude(Number_of_Y_Dimension, Number_of_X_Dimension) ; + Longitude:long_name = "Longitude for EASE-2 grid" ; + Longitude:units = "degrees_east" ; + Longitude:valid_range = -180.f, 180.f ; + Longitude:_FillValue = -9999.f ; + Longitude:standard_name = "longitude" ; + int NASA_Team_2_Ice_Concentration(Number_of_Y_Dimension, Number_of_X_Dimension) ; + NASA_Team_2_Ice_Concentration:long_name = "NASA Team 2 Ice Concentration (Fraction) - primary product" ; + NASA_Team_2_Ice_Concentration:units = "1" ; + NASA_Team_2_Ice_Concentration:coordinates = "Latitude Longitude" ; + NASA_Team_2_Ice_Concentration:_FillValue = -9999 ; + int NASA_Team_2_Multiyear_Ice(Number_of_Y_Dimension, Number_of_X_Dimension) ; + NASA_Team_2_Multiyear_Ice:long_name = "NASA Team 2 Multiyear Ice (Fraction)" ; + NASA_Team_2_Multiyear_Ice:units = "1" ; + NASA_Team_2_Multiyear_Ice:coordinates = "Latitude Longitude" ; + NASA_Team_2_Multiyear_Ice:_FillValue = -9999 ; + int NT2_minus_Bootstrap(Number_of_Y_Dimension, Number_of_X_Dimension) ; + NT2_minus_Bootstrap:long_name = "NT2 minus Bootstrap (fraction)" ; + NT2_minus_Bootstrap:units = "1" ; + NT2_minus_Bootstrap:coordinates = "Latitude Longitude" ; + NT2_minus_Bootstrap:_FillValue = -9999 ; + float QM_Num_Grid_Range_25(QM) ; + QM_Num_Grid_Range_25:long_name = "Quality Monitoring: Number of Grid Boxes with Range > 25%" ; + QM_Num_Grid_Range_25:units = "1" ; + QM_Num_Grid_Range_25:_FillValue = -9999.f ; + float QM_Num_Grid_Range_50(QM) ; + QM_Num_Grid_Range_50:long_name = "Quality Monitoring: Number of Grid Boxes with Range > 50%" ; + QM_Num_Grid_Range_50:units = "1" ; + QM_Num_Grid_Range_50:_FillValue = -9999.f ; + float QM_Total_Ice(QM) ; + QM_Total_Ice:long_name = "Quality Monitoring: Total Ice Extent" ; + QM_Total_Ice:units = "1" ; + QM_Total_Ice:_FillValue = -9999.f ; + float QM_Total_Pixels(QM) ; + QM_Total_Pixels:long_name = "Quality Monitoring: Total Number of Pixels" ; + QM_Total_Pixels:units = "1" ; + QM_Total_Pixels:_FillValue = -9999.f ; + int Range_of_Ice_Concentration(Number_of_Y_Dimension, Number_of_X_Dimension) ; + Range_of_Ice_Concentration:long_name = "Range of Ice Concentration (Fraction) - based on NT2" ; + Range_of_Ice_Concentration:units = "1" ; + Range_of_Ice_Concentration:coordinates = "Latitude Longitude" ; + Range_of_Ice_Concentration:_FillValue = -9999 ; + float Scan_Time(Number_of_Y_Dimension, Number_of_X_Dimension, Time_Dimension) ; + Scan_Time:long_name = "Scan line Start Time 6-D for (YYYY, MM, DD, HH, MM, SS) in GMT" ; + Scan_Time:units = "1" ; + Scan_Time:_FillValue = -9999.f ; + +// global attributes: + :Conventions = "CF-1.5" ; + :Metadata_Conventions = "CF-1.5, Unidata Datasset Discovery v1.0" ; + :standard_name_vocabulary = "CF Standard Name Table (version 17, 24 March 2011)" ; + :project = "NPP Data Exploitation: NOAA GCOM-W1 AMSR2" ; + :title = "AMSR2_SEAICE" ; + :summary = "GCOM Seaice Products" ; + :institution = "DOC/NOAA/NESDIS/OSPO > Office of Satellite and Product Operations, NESDIS, NOAA, U.S. Department of Commerce" ; + :naming_authority = "gov.noaa.nesdis.nde" ; + :platform_name = "GCOM-W1" ; + :instrument_name = "AMSR2" ; + :creator_name = "DOC/NOAA/NESDIS/STAR > IOSSPDT Algorithm Team, Center for Satellite Applications and Research, NESDIS, NOAA, U.S. Department of Commerce" ; + :creator_email = "espcoperations@noaa.gov" ; + :creator_url = "http://www.star.nesdis.noaa.gov" ; + :publisher_name = "DOC/NOAA/NESDIS/NDE > NPP Data Exploitation, Center for Satellite Applications and Research, NESDIS, NOAA, U.S. Department of Commerce" ; + :publisher_email = "espcoperations@noaa.gov" ; + :publisher_url = "http://www.ospo.noaa.gov/" ; + :references = "Contact the OSPO PAL to request the ATBD." ; + :processing_level = "NOAA Level 2 data" ; + :cdm_data_type = "Grid" ; + :geospatial_lat_min = 18.87699f ; + :geospatial_lat_max = 89.9367f ; + :geospatial_lon_min = -179.945f ; + :geospatial_lon_max = 179.945f ; + :geospatial_lat_units = "degrees_north" ; + :geospatial_lon_units = "degrees_east" ; + :date_created = "2021-07-01T02:02:39Z" ; + :id = "abb20bb6-945f-4f02-a7a5-14f6dcdce4cf" ; + :Metadata_Link = "AMSR2-SEAICE-NH_v2r2_GW1_s202106302341190_e202107010120160_c202107010202390.nc" ; + :history = "Mon Sep 25 15:10:33 2023: ncks -d Number_of_X_Dimension,400,600,100 -d Number_of_Y_Dimension,400,700,100 /scratch1/NCEPDEV/stmp4/Shastri.Paturi/forAndrew/gdas.20210701/00/icec/AMSR2-SEAICE-NH_v2r2_GW1_s202106302341190_e202107010120160_c202107010202390.nc output.nc\nCreated by GAASP Version 2.0, Release 2.0" ; + :source = "GAASP-L1R_v2r2_GW1_s202106302341190_e202107010120160_c202107010159060.h5, GAASP-L1R_v2r2_GW1_s202106302202190_e202106302341170_c202107010019490.h5, GAASP-L1R_v2r2_GW1_s202106302020180_e202106302202180_c202106302237500.h5, GAASP-L1R_v2r2_GW1_s202106301838190_e202106302020170_c202106302057270.h5, GAASP-L1R_v2r2_GW1_s202106301659190_e202106301838170_c202106301916000.h5, GAASP-L1R_v2r2_GW1_s202106301520170_e202106301659180_c202106301736190.h5, GAASP-L1R_v2r2_GW1_s202106301341160_e202106301520150_c202106301558020.h5, GAASP-L1R_v2r2_GW1_s202106301205160_e202106301341140_c202106301416190.h5, GAASP-L1R_v2r2_GW1_s202106301026170_e202106301205150_c202106301238160.h5, GAASP-L1R_v2r2_GW1_s202106300847160_e202106301026160_c202106301101410.h5, GAASP-L1R_v2r2_GW1_s202106300711170_e202106300847150_c202106300921380.h5, GAASP-L1R_v2r2_GW1_s202106300532170_e202106300711150_c202106300744420.h5, GAASP-L1R_v2r2_GW1_s202106300356160_e202106300532160_c202106300607150.h5, GAASP-L1R_v2r2_GW1_s202106300217170_e202106300356150_c202106300428400.h5, GAASP-L1R_v2r2_GW1_s202106300038170_e202106300217150_c202106300252420.h5, GAASP-L1R_v2r2_GW1_s202106292259160_e202106300038160_c202106300112300.h5" ; + :production_site = "NSOF" ; + :production_environment = "OE" ; + :start_orbit_number = 48513 ; + :end_orbit_number = 48514 ; + :day_night_data_flag = 2 ; + :ascend_descend_data_flag = 2 ; + :time_coverage_start = "2021-06-30T23:41:19.004Z" ; + :time_coverage_end = "2021-07-01T01:20:16.940Z" ; + :NCO = "netCDF Operators version 5.0.6 (Homepage = http://nco.sf.net, Code = http://github.com/nco/nco)" ; +data: + + Across_X_Dimension = 401, 501, 601 ; + + Along_Y_Dimension = 401, 501, 601, 701 ; + + Bootstrap_Ice_Concentration = + 100, 100, 0, + _, 100, 100, + _, 87, 96, + _, 0, 0 ; + + Flags = + 0, 0, 8, + 120, 0, 0, + 120, 0, 0, + 120, 8, 8 ; + + Latency = + 204, 104, 6, + 2000, 299, 3, + 2000, 696, 1, + 2000, 699, 897 ; + + Latitude = + 74.18227, 78.6195, 76.93348, + 78.6195, 86.89752, 82.88853, + 76.93348, 82.88853, 80.42855, + 70.63603, 74.07984, 72.82585 ; + + Longitude = + -135, -168.8672, 148.7663, + -101.1328, -135, 107.9784, + -58.7663, -17.97842, 45, + -35.35196, -7.947196, 23.27735 ; + + NASA_Team_2_Ice_Concentration = + 100, 100, 0, + _, 100, 100, + _, 97, 100, + _, 0, 0 ; + + NASA_Team_2_Multiyear_Ice = + 0, 0, 0, + _, 100, 100, + _, 0, 0, + _, 0, 0 ; + + NT2_minus_Bootstrap = + 0, 0, 0, + _, 0, 0, + _, 10, 4, + _, 0, 0 ; + + QM_Num_Grid_Range_25 = 10331 ; + + QM_Num_Grid_Range_50 = 2560 ; + + QM_Total_Ice = 98806 ; + + QM_Total_Pixels = 1102500 ; + + Range_of_Ice_Concentration = + 1, 3, 0, + _, 0, 3, + _, 13, 4, + _, 0, 0 ; + + Scan_Time = + 2018, 4, 15, 9, 55, 22.60762, + 2018, 4, 15, 9, 35, 23.53783, + 2018, 4, 15, 9, 13, 57.47626, + _, _, _, _, _, _, + 2018, 4, 15, 9, 20, 29.15068, + 2018, 4, 15, 10, 16, 25.96225, + _, _, _, _, _, _, + 2018, 4, 15, 11, 43, 40.417, + 2018, 4, 15, 12, 18, 54.4482, + _, _, _, _, _, _, + 2018, 4, 15, 13, 41, 11.93098, + 2018, 4, 15, 14, 22, 37.06214 ; +} diff --git a/test/soca/testdata/icec_amsr2_north_2.cdl b/test/soca/testdata/icec_amsr2_north_2.cdl new file mode 100644 index 000000000..afb71f57f --- /dev/null +++ b/test/soca/testdata/icec_amsr2_north_2.cdl @@ -0,0 +1,208 @@ +netcdf icec_amsr2_north_2 { +dimensions: + Number_of_X_Dimension = 3 ; + Number_of_Y_Dimension = 4 ; + QM = 1 ; + Time_Dimension = 6 ; +variables: + short Across_X_Dimension(Number_of_X_Dimension) ; + Across_X_Dimension:long_name = " Across X Dimension" ; + Across_X_Dimension:coordinates = "" ; + Across_X_Dimension:units = "1" ; + Across_X_Dimension:_FillValue = -9999s ; + short Along_Y_Dimension(Number_of_Y_Dimension) ; + Along_Y_Dimension:long_name = " Along Y Dimension" ; + Along_Y_Dimension:coordinates = "" ; + Along_Y_Dimension:units = "1" ; + Along_Y_Dimension:_FillValue = -9999s ; + int Bootstrap_Ice_Concentration(Number_of_Y_Dimension, Number_of_X_Dimension) ; + Bootstrap_Ice_Concentration:long_name = "Bootstrap Ice Concentration (Fraction) - secondary product" ; + Bootstrap_Ice_Concentration:units = "1" ; + Bootstrap_Ice_Concentration:coordinates = "Latitude Longitude" ; + Bootstrap_Ice_Concentration:_FillValue = -9999 ; + short Flags(Number_of_Y_Dimension, Number_of_X_Dimension) ; + Flags:long_name = "Quality Flags" ; + Flags:units = "1" ; + Flags:coordinate = "Latitude Longitude" ; + Flags:_FillValue = -9999s ; + Flags:comment = "0: good, 4: SST missing, 8: Weather, 16: spillover, 32: filled, 64: NT2 Missing" ; + int Latency(Number_of_Y_Dimension, Number_of_X_Dimension) ; + Latency:long_name = "Time in seconds from most current time" ; + Latency:units = "seconds" ; + Latency:coordinate = "Latitude Longitude" ; + Latency:_FillValue = -9999 ; + float Latitude(Number_of_Y_Dimension, Number_of_X_Dimension) ; + Latitude:long_name = "Latitude for EASE-2 grid" ; + Latitude:units = "degrees_north" ; + Latitude:valid_range = -90.f, 90.f ; + Latitude:_FillValue = -9999.f ; + Latitude:standard_name = "latitude" ; + float Longitude(Number_of_Y_Dimension, Number_of_X_Dimension) ; + Longitude:long_name = "Longitude for EASE-2 grid" ; + Longitude:units = "degrees_east" ; + Longitude:valid_range = -180.f, 180.f ; + Longitude:_FillValue = -9999.f ; + Longitude:standard_name = "longitude" ; + int NASA_Team_2_Ice_Concentration(Number_of_Y_Dimension, Number_of_X_Dimension) ; + NASA_Team_2_Ice_Concentration:long_name = "NASA Team 2 Ice Concentration (Fraction) - primary product" ; + NASA_Team_2_Ice_Concentration:units = "1" ; + NASA_Team_2_Ice_Concentration:coordinates = "Latitude Longitude" ; + NASA_Team_2_Ice_Concentration:_FillValue = -9999 ; + int NASA_Team_2_Multiyear_Ice(Number_of_Y_Dimension, Number_of_X_Dimension) ; + NASA_Team_2_Multiyear_Ice:long_name = "NASA Team 2 Multiyear Ice (Fraction)" ; + NASA_Team_2_Multiyear_Ice:units = "1" ; + NASA_Team_2_Multiyear_Ice:coordinates = "Latitude Longitude" ; + NASA_Team_2_Multiyear_Ice:_FillValue = -9999 ; + int NT2_minus_Bootstrap(Number_of_Y_Dimension, Number_of_X_Dimension) ; + NT2_minus_Bootstrap:long_name = "NT2 minus Bootstrap (fraction)" ; + NT2_minus_Bootstrap:units = "1" ; + NT2_minus_Bootstrap:coordinates = "Latitude Longitude" ; + NT2_minus_Bootstrap:_FillValue = -9999 ; + float QM_Num_Grid_Range_25(QM) ; + QM_Num_Grid_Range_25:long_name = "Quality Monitoring: Number of Grid Boxes with Range > 25%" ; + QM_Num_Grid_Range_25:units = "1" ; + QM_Num_Grid_Range_25:_FillValue = -9999.f ; + float QM_Num_Grid_Range_50(QM) ; + QM_Num_Grid_Range_50:long_name = "Quality Monitoring: Number of Grid Boxes with Range > 50%" ; + QM_Num_Grid_Range_50:units = "1" ; + QM_Num_Grid_Range_50:_FillValue = -9999.f ; + float QM_Total_Ice(QM) ; + QM_Total_Ice:long_name = "Quality Monitoring: Total Ice Extent" ; + QM_Total_Ice:units = "1" ; + QM_Total_Ice:_FillValue = -9999.f ; + float QM_Total_Pixels(QM) ; + QM_Total_Pixels:long_name = "Quality Monitoring: Total Number of Pixels" ; + QM_Total_Pixels:units = "1" ; + QM_Total_Pixels:_FillValue = -9999.f ; + int Range_of_Ice_Concentration(Number_of_Y_Dimension, Number_of_X_Dimension) ; + Range_of_Ice_Concentration:long_name = "Range of Ice Concentration (Fraction) - based on NT2" ; + Range_of_Ice_Concentration:units = "1" ; + Range_of_Ice_Concentration:coordinates = "Latitude Longitude" ; + Range_of_Ice_Concentration:_FillValue = -9999 ; + float Scan_Time(Number_of_Y_Dimension, Number_of_X_Dimension, Time_Dimension) ; + Scan_Time:long_name = "Scan line Start Time 6-D for (YYYY, MM, DD, HH, MM, SS) in GMT" ; + Scan_Time:units = "1" ; + Scan_Time:_FillValue = -9999.f ; + +// global attributes: + :Conventions = "CF-1.5" ; + :Metadata_Conventions = "CF-1.5, Unidata Datasset Discovery v1.0" ; + :standard_name_vocabulary = "CF Standard Name Table (version 17, 24 March 2011)" ; + :project = "NPP Data Exploitation: NOAA GCOM-W1 AMSR2" ; + :title = "AMSR2_SEAICE" ; + :summary = "GCOM Seaice Products" ; + :institution = "DOC/NOAA/NESDIS/OSPO > Office of Satellite and Product Operations, NESDIS, NOAA, U.S. Department of Commerce" ; + :naming_authority = "gov.noaa.nesdis.nde" ; + :platform_name = "GCOM-W1" ; + :instrument_name = "AMSR2" ; + :creator_name = "DOC/NOAA/NESDIS/STAR > IOSSPDT Algorithm Team, Center for Satellite Applications and Research, NESDIS, NOAA, U.S. Department of Commerce" ; + :creator_email = "espcoperations@noaa.gov" ; + :creator_url = "http://www.star.nesdis.noaa.gov" ; + :publisher_name = "DOC/NOAA/NESDIS/NDE > NPP Data Exploitation, Center for Satellite Applications and Research, NESDIS, NOAA, U.S. Department of Commerce" ; + :publisher_email = "espcoperations@noaa.gov" ; + :publisher_url = "http://www.ospo.noaa.gov/" ; + :references = "Contact the OSPO PAL to request the ATBD." ; + :processing_level = "NOAA Level 2 data" ; + :cdm_data_type = "Grid" ; + :geospatial_lat_min = 18.87699f ; + :geospatial_lat_max = 89.9367f ; + :geospatial_lon_min = -179.945f ; + :geospatial_lon_max = 179.945f ; + :geospatial_lat_units = "degrees_north" ; + :geospatial_lon_units = "degrees_east" ; + :date_created = "2021-07-01T03:40:15Z" ; + :id = "84a33a48-3a6d-43f6-bb82-48e9072799a9" ; + :Metadata_Link = "AMSR2-SEAICE-NH_v2r2_GW1_s202107010120180_e202107010259170_c202107010340150.nc" ; + :history = "Mon Sep 25 15:41:32 2023: ncks -d Number_of_X_Dimension,400,600,100 -d Number_of_Y_Dimension,400,700,100 /scratch1/NCEPDEV/stmp4/Shastri.Paturi/forAndrew/gdas.20210701/00/icec/AMSR2-SEAICE-NH_v2r2_GW1_s202107010120180_e202107010259170_c202107010340150.nc icec_amsr2_north_2.nc\nCreated by GAASP Version 2.0, Release 2.0" ; + :source = "GAASP-L1R_v2r2_GW1_s202107010120180_e202107010259170_c202107010336520.h5, GAASP-L1R_v2r2_GW1_s202106302341190_e202107010120160_c202107010159060.h5, GAASP-L1R_v2r2_GW1_s202106302202190_e202106302341170_c202107010019490.h5, GAASP-L1R_v2r2_GW1_s202106302020180_e202106302202180_c202106302237500.h5, GAASP-L1R_v2r2_GW1_s202106301838190_e202106302020170_c202106302057270.h5, GAASP-L1R_v2r2_GW1_s202106301659190_e202106301838170_c202106301916000.h5, GAASP-L1R_v2r2_GW1_s202106301520170_e202106301659180_c202106301736190.h5, GAASP-L1R_v2r2_GW1_s202106301341160_e202106301520150_c202106301558020.h5, GAASP-L1R_v2r2_GW1_s202106301205160_e202106301341140_c202106301416190.h5, GAASP-L1R_v2r2_GW1_s202106301026170_e202106301205150_c202106301238160.h5, GAASP-L1R_v2r2_GW1_s202106300847160_e202106301026160_c202106301101410.h5, GAASP-L1R_v2r2_GW1_s202106300711170_e202106300847150_c202106300921380.h5, GAASP-L1R_v2r2_GW1_s202106300532170_e202106300711150_c202106300744420.h5, GAASP-L1R_v2r2_GW1_s202106300356160_e202106300532160_c202106300607150.h5, GAASP-L1R_v2r2_GW1_s202106300217170_e202106300356150_c202106300428400.h5, GAASP-L1R_v2r2_GW1_s202106300038170_e202106300217150_c202106300252420.h5" ; + :production_site = "NSOF" ; + :production_environment = "OE" ; + :start_orbit_number = 48514 ; + :end_orbit_number = 48515 ; + :day_night_data_flag = 2 ; + :ascend_descend_data_flag = 2 ; + :time_coverage_start = "2021-07-01T01:20:18.440Z" ; + :time_coverage_end = "2021-07-01T02:59:17.876Z" ; + :NCO = "netCDF Operators version 5.0.6 (Homepage = http://nco.sf.net, Code = http://github.com/nco/nco)" ; +data: + + Across_X_Dimension = 401, 501, 601 ; + + Along_Y_Dimension = 401, 501, 601, 701 ; + + Bootstrap_Ice_Concentration = + 100, 95, 0, + _, 100, 100, + _, 87, 95, + _, 0, 0 ; + + Flags = + 0, 0, 8, + 120, 0, 0, + 120, 0, 0, + 120, 8, 4 ; + + Latency = + 303, 203, 6, + 2000, 398, 4, + 2000, 795, 2, + 2000, 798, _ ; + + Latitude = + 74.18227, 78.6195, 76.93348, + 78.6195, 86.89752, 82.88853, + 76.93348, 82.88853, 80.42855, + 70.63603, 74.07984, 72.82585 ; + + Longitude = + -135, -168.8672, 148.7663, + -101.1328, -135, 107.9784, + -58.7663, -17.97842, 45, + -35.35196, -7.947196, 23.27735 ; + + NASA_Team_2_Ice_Concentration = + 100, 100, 0, + _, 100, 97, + _, 97, 100, + _, 0, 0 ; + + NASA_Team_2_Multiyear_Ice = + 0, 79, 0, + _, 100, 100, + _, 0, 0, + _, 0, 0 ; + + NT2_minus_Bootstrap = + 0, 5, 0, + _, 0, -2, + _, 10, 5, + _, 0, 0 ; + + QM_Num_Grid_Range_25 = 10266 ; + + QM_Num_Grid_Range_50 = 2384 ; + + QM_Total_Ice = 98282 ; + + QM_Total_Pixels = 1102500 ; + + Range_of_Ice_Concentration = + 1, 3, 0, + _, 0, 3, + _, 13, 4, + _, 0, 0 ; + + Scan_Time = + 2018, 4, 15, 9, 55, 22.60762, + 2018, 4, 15, 9, 35, 23.53783, + 2018, 4, 15, 10, 52, 49.41316, + _, _, _, _, _, _, + 2018, 4, 15, 11, 20, 29.15068, + 2018, 4, 15, 12, 54, 38.90287, + _, _, _, _, _, _, + 2018, 4, 15, 13, 43, 40.417, + 2018, 4, 15, 13, 56, 49.3905, + _, _, _, _, _, _, + 2018, 4, 15, 14, 41, 11.93098, + _, _, _, _, _, _ ; +} diff --git a/test/soca/testdata/icec_amsr2_south_1.cdl b/test/soca/testdata/icec_amsr2_south_1.cdl new file mode 100644 index 000000000..7d19a1c74 --- /dev/null +++ b/test/soca/testdata/icec_amsr2_south_1.cdl @@ -0,0 +1,306 @@ +netcdf icec_amsr2_south { +dimensions: + Number_of_X_Dimension = 8 ; + Number_of_Y_Dimension = 8 ; + QM = 1 ; + Time_Dimension = 6 ; +variables: + short Across_X_Dimension(Number_of_X_Dimension) ; + Across_X_Dimension:long_name = " Across X Dimension" ; + Across_X_Dimension:coordinates = "" ; + Across_X_Dimension:units = "1" ; + Across_X_Dimension:_FillValue = -9999s ; + short Along_Y_Dimension(Number_of_Y_Dimension) ; + Along_Y_Dimension:long_name = " Along Y Dimension" ; + Along_Y_Dimension:coordinates = "" ; + Along_Y_Dimension:units = "1" ; + Along_Y_Dimension:_FillValue = -9999s ; + int Bootstrap_Ice_Concentration(Number_of_Y_Dimension, Number_of_X_Dimension) ; + Bootstrap_Ice_Concentration:long_name = "Bootstrap Ice Concentration (Fraction)" ; + Bootstrap_Ice_Concentration:units = "1" ; + Bootstrap_Ice_Concentration:coordinates = "Latitude Longitude" ; + Bootstrap_Ice_Concentration:_FillValue = -9999 ; + short Flags(Number_of_Y_Dimension, Number_of_X_Dimension) ; + Flags:long_name = "Quality Flags" ; + Flags:units = "1" ; + Flags:coordinate = "Latitude Longitude" ; + Flags:_FillValue = -9999s ; + Flags:comment = "0: good, 4: SST missing, 8: Weather, 16: spillover, 32: filled, 64: NT2 Missing" ; + int Latency(Number_of_Y_Dimension, Number_of_X_Dimension) ; + Latency:long_name = "Time in seconds from most current time" ; + Latency:units = "seconds" ; + Latency:coordinate = "Latitude Longitude" ; + Latency:_FillValue = -9999 ; + float Latitude(Number_of_Y_Dimension, Number_of_X_Dimension) ; + Latitude:long_name = "Latitude for EASE-2 grid" ; + Latitude:units = "degrees_north" ; + Latitude:valid_range = -90.f, 90.f ; + Latitude:_FillValue = -9999.f ; + Latitude:standard_name = "latitude" ; + float Longitude(Number_of_Y_Dimension, Number_of_X_Dimension) ; + Longitude:long_name = "Longitude for EASE-2 grid" ; + Longitude:units = "degrees_east" ; + Longitude:valid_range = -180.f, 180.f ; + Longitude:_FillValue = -9999.f ; + Longitude:standard_name = "longitude" ; + int NASA_Team_2_Ice_Concentration(Number_of_Y_Dimension, Number_of_X_Dimension) ; + NASA_Team_2_Ice_Concentration:long_name = "NASA Team 2 Ice Concentration (Fraction)" ; + NASA_Team_2_Ice_Concentration:units = "1" ; + NASA_Team_2_Ice_Concentration:coordinates = "Latitude Longitude" ; + NASA_Team_2_Ice_Concentration:_FillValue = -9999 ; + int NASA_Team_2_Multiyear_Ice(Number_of_Y_Dimension, Number_of_X_Dimension) ; + NASA_Team_2_Multiyear_Ice:long_name = "NASA Team 2 Multiyear Ice (Fraction)" ; + NASA_Team_2_Multiyear_Ice:units = "1" ; + NASA_Team_2_Multiyear_Ice:coordinates = "Latitude Longitude" ; + NASA_Team_2_Multiyear_Ice:_FillValue = -9999 ; + int NT2_minus_Bootstrap(Number_of_Y_Dimension, Number_of_X_Dimension) ; + NT2_minus_Bootstrap:long_name = "NT2 minus Bootstrap (Fraction)" ; + NT2_minus_Bootstrap:units = "1" ; + NT2_minus_Bootstrap:coordinates = "Latitude Longitude" ; + NT2_minus_Bootstrap:_FillValue = -9999 ; + float QM_Num_Grid_Range_25(QM) ; + QM_Num_Grid_Range_25:long_name = "Quality Monitoring: Number of Grid Boxes with Range > 25%" ; + QM_Num_Grid_Range_25:units = "1" ; + QM_Num_Grid_Range_25:_FillValue = -9999.f ; + float QM_Num_Grid_Range_50(QM) ; + QM_Num_Grid_Range_50:long_name = "Quality Monitoring: Number of Grid Boxes with Range > 50%" ; + QM_Num_Grid_Range_50:units = "1" ; + QM_Num_Grid_Range_50:_FillValue = -9999.f ; + float QM_Total_Ice(QM) ; + QM_Total_Ice:long_name = "Quality Monitoring: Total Ice Extent" ; + QM_Total_Ice:units = "1" ; + QM_Total_Ice:_FillValue = -9999.f ; + float QM_Total_Pixels(QM) ; + QM_Total_Pixels:long_name = "Quality Monitoring: Total Number of Pixels" ; + QM_Total_Pixels:units = "1" ; + QM_Total_Pixels:_FillValue = -9999.f ; + int Range_of_Ice_Concentration(Number_of_Y_Dimension, Number_of_X_Dimension) ; + Range_of_Ice_Concentration:long_name = "Range of Ice Concentration (Fraction)" ; + Range_of_Ice_Concentration:units = "1" ; + Range_of_Ice_Concentration:coordinates = "Latitude Longitude" ; + Range_of_Ice_Concentration:_FillValue = -9999 ; + float Scan_Time(Number_of_Y_Dimension, Number_of_X_Dimension, Time_Dimension) ; + Scan_Time:long_name = "Scan line Start Time 6-D for (YYYY, MM, DD, HH, MM, SS) in GMT" ; + Scan_Time:units = "1" ; + Scan_Time:_FillValue = -9999.f ; + +// global attributes: + :Conventions = "CF-1.5" ; + :Metadata_Conventions = "CF-1.5, Unidata Datasset Discovery v1.0" ; + :standard_name_vocabulary = "CF Standard Name Table (version 17, 24 March 2011)" ; + :project = "NPP Data Exploitation: NOAA GCOM-W1 AMSR2" ; + :title = "AMSR2_SEAICE_SH" ; + :summary = "GCOM Seaice Southern Hemisphere Products" ; + :institution = "DOC/NOAA/NESDIS/OSPO > Office of Satellite and Product Operations, NESDIS, NOAA, U.S. Department of Commerce" ; + :naming_authority = "gov.noaa.nesdis.nde" ; + :platform_name = "GCOM-W1" ; + :instrument_name = "AMSR2" ; + :creator_name = "DOC/NOAA/NESDIS/STAR > IOSSPDT Algorithm Team, Center for Satellite Applications and Research, NESDIS, NOAA, U.S. Department of Commerce" ; + :creator_email = "espcoperations@noaa.gov" ; + :creator_url = "http://www.star.nesdis.noaa.gov" ; + :publisher_name = "DOC/NOAA/NESDIS/NDE > NPP Data Exploitation, Center for Satellite Applications and Research, NESDIS, NOAA, U.S. Department of Commerce" ; + :publisher_email = "espcoperations@noaa.gov" ; + :publisher_url = "http://www.ospo.noaa.gov/" ; + :references = "Contact the OSPO PAL to request the ATBD." ; + :processing_level = "NOAA Level 2 data" ; + :cdm_data_type = "Grid" ; + :geospatial_lat_min = -89.9367f ; + :geospatial_lat_max = -34.6223f ; + :geospatial_lon_min = -179.932f ; + :geospatial_lon_max = 179.932f ; + :geospatial_lat_units = "degrees_north" ; + :geospatial_lon_units = "degrees_east" ; + :date_created = "2021-07-01T02:02:39Z" ; + :id = "09bd3374-b3bf-46e4-b4bc-f698cccbd717" ; + :Metadata_Link = "AMSR2-SEAICE-SH_v2r2_GW1_s202106302341190_e202107010120160_c202107010202390.nc" ; + :history = "Mon Sep 25 15:26:46 2023: ncks -d Number_of_X_Dimension,100,800,100 -d Number_of_Y_Dimension,100,800,100 /scratch1/NCEPDEV/stmp4/Shastri.Paturi/forAndrew/gdas.20210701/00/icec/AMSR2-SEAICE-SH_v2r2_GW1_s202106302341190_e202107010120160_c202107010202390.nc icec_amsr2_south.nc\nCreated by GAASP Version 2.0, Release 2.0" ; + :source = "GAASP-L1R_v2r2_GW1_s202106302341190_e202107010120160_c202107010159060.h5, GAASP-L1R_v2r2_GW1_s202106302202190_e202106302341170_c202107010019490.h5, GAASP-L1R_v2r2_GW1_s202106302020180_e202106302202180_c202106302237500.h5, GAASP-L1R_v2r2_GW1_s202106301838190_e202106302020170_c202106302057270.h5, GAASP-L1R_v2r2_GW1_s202106301659190_e202106301838170_c202106301916000.h5, GAASP-L1R_v2r2_GW1_s202106301520170_e202106301659180_c202106301736190.h5, GAASP-L1R_v2r2_GW1_s202106301341160_e202106301520150_c202106301558020.h5, GAASP-L1R_v2r2_GW1_s202106301205160_e202106301341140_c202106301416190.h5, GAASP-L1R_v2r2_GW1_s202106301026170_e202106301205150_c202106301238160.h5, GAASP-L1R_v2r2_GW1_s202106300847160_e202106301026160_c202106301101410.h5, GAASP-L1R_v2r2_GW1_s202106300711170_e202106300847150_c202106300921380.h5, GAASP-L1R_v2r2_GW1_s202106300532170_e202106300711150_c202106300744420.h5, GAASP-L1R_v2r2_GW1_s202106300356160_e202106300532160_c202106300607150.h5, GAASP-L1R_v2r2_GW1_s202106300217170_e202106300356150_c202106300428400.h5, GAASP-L1R_v2r2_GW1_s202106300038170_e202106300217150_c202106300252420.h5, GAASP-L1R_v2r2_GW1_s202106292259160_e202106300038160_c202106300112300.h5" ; + :production_site = "NSOF" ; + :production_environment = "OE" ; + :time_coverage_start = "2021-06-30T23:41:19.004Z" ; + :time_coverage_end = "2021-07-01T01:20:16.940Z" ; + :start_orbit_number = 48513 ; + :end_orbit_number = 48514 ; + :ascend_descend_data_flag = 2 ; + :day_night_data_flag = 2 ; + :NCO = "netCDF Operators version 5.0.6 (Homepage = http://nco.sf.net, Code = http://github.com/nco/nco)" ; +data: + + Across_X_Dimension = 101, 201, 301, 401, 501, 601, 701, 801 ; + + Along_Y_Dimension = 101, 201, 301, 401, 501, 601, 701, 801 ; + + Bootstrap_Ice_Concentration = + 0, 0, 0, 100, 93, 0, 0, 0, + 0, 94, 100, 91, 95, 84, 0, 0, + 0, _, 100, _, _, _, 97, 0, + 0, 100, _, _, _, _, 100, 0, + 0, 99, _, _, _, _, 95, 0, + 0, 0, 100, 96, _, _, 0, 0, + 0, 0, 0, 0, 91, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0 ; + + Flags = + 4, 8, 8, 0, 0, 8, 8, 4, + 4, 0, 0, 0, 0, 0, 8, 8, + 8, 120, 0, 120, 120, 120, 0, 8, + 8, 0, 120, 120, 120, 120, 0, 8, + 8, 0, 120, 120, 120, 120, 0, 8, + 4, 8, 0, 0, 120, 120, 8, 4, + 4, 8, 8, 8, 0, 8, 8, 4, + 4, 4, 4, 4, 4, 4, 4, 4 ; + + Latency = + _, 537, 539, 60, 60, 160, 162, _, + _, 441, 442, 57, 157, 158, 259, 261, + 341, 2000, 54, 2000, 2000, 2000, 257, 358, + 242, 245, 2000, 2000, 2000, 2000, 356, 358, + 242, 146, 2000, 2000, 2000, 2000, 455, 1032, + _, 45, 47, 748, 2000, 2000, 555, _, + _, 44, 45, 751, 652, 653, 557, _, + _, _, _, _, _, _, _, _ ; + + Latitude = + -48.58855, -54.69889, -59.05717, -61.01042, -60.13906, -56.64225, + -51.14237, -44.22784, + -54.69889, -61.90622, -67.46837, -70.16405, -68.94062, -64.3263, -57.65202, + -49.79427, + -59.05717, -67.46837, -74.82165, -79.14207, -77.07041, -70.51806, + -62.41702, -53.64244, + -61.01042, -70.16405, -79.14207, -87.5308, -82.57877, -73.68634, -64.60285, + -55.3238, + -60.13906, -68.94062, -77.07041, -82.57877, -79.79292, -72.22871, + -63.62301, -54.57748, + -56.64225, -64.3263, -70.51806, -73.68634, -72.22871, -66.97952, -59.76062, + -51.52504, + -51.14237, -57.65202, -62.41702, -64.60285, -63.62301, -59.76062, + -53.84447, -46.57473, + -44.22784, -49.79427, -53.64244, -55.3238, -54.57748, -51.52504, -46.57473, + -40.16538 ; + + Longitude = + -45, -34.48949, -20.5069, -3.492593, 14.14168, 29.46407, 41.28101, 49.98034, + -55.51051, -45, -28.56472, -5.07673, 20.14014, 39.43126, 51.95571, 60.02053, + -69.4931, -61.43528, -45, -9.267837, 33.96579, 56.49345, 66.92478, 72.56456, + -86.50741, -84.92327, -80.73216, -45, 76.38319, 83.83407, 86.02327, 87.06625, + -104.1417, -110.1401, -123.9658, -166.3832, 135, 114.0361, 106.0128, + 101.9456, + -119.4641, -129.4313, -146.4935, -173.8341, 155.9639, 135, 122.761, 115.3785, + -131.281, -141.9557, -156.9248, -176.0233, 163.9872, 147.239, 135, 126.3972, + -139.9803, -150.0205, -162.5646, -177.0663, 168.0544, 154.6215, 143.6028, + 135 ; + + NASA_Team_2_Ice_Concentration = + 0, 0, 0, 100, 100, 0, 0, 0, + 0, 100, 100, 100, 100, 99, 0, 0, + 0, _, 100, _, _, _, 100, 0, + 0, 99, _, _, _, _, 100, 0, + 0, 99, _, _, _, _, 100, 0, + 0, 0, 100, 100, _, _, 0, 0, + 0, 0, 0, 0, 100, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0 ; + + NASA_Team_2_Multiyear_Ice = + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, _, 0, _, _, _, 0, 0, + 0, 0, _, _, _, _, 0, 0, + 0, 65, _, _, _, _, 0, 0, + 0, 0, 0, 0, _, _, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0 ; + + NT2_minus_Bootstrap = + 0, 0, 0, 0, 7, 0, 0, 0, + 0, 6, 0, 9, 5, 15, 0, 0, + 0, _, 0, _, _, _, 3, 0, + 0, 0, _, _, _, _, 0, 0, + 0, 0, _, _, _, _, 5, 0, + 0, 0, 0, 4, _, _, 0, 0, + 0, 0, 0, 0, 9, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0 ; + + QM_Num_Grid_Range_25 = 3427 ; + + QM_Num_Grid_Range_50 = 830 ; + + QM_Total_Ice = 148634 ; + + QM_Total_Pixels = 705600 ; + + Range_of_Ice_Concentration = + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1, 2, 0, 0, 1, 0, 0, + 0, _, 0, _, _, _, 0, 0, + 0, 1, _, _, _, _, 0, 0, + 0, 4, _, _, _, _, 0, 0, + 0, 0, 0, 0, _, _, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0 ; + + Scan_Time = + _, _, _, _, _, _, + 2018, 4, 15, 16, 22, 42.51182, + 2018, 4, 15, 16, 21, 9.52068, + 2018, 4, 15, 0, 20, 8.284265, + 2018, 4, 15, 0, 19, 36.78727, + 2018, 4, 15, 22, 39, 34.35703, + 2018, 4, 15, 22, 38, 11.86481, + _, _, _, _, _, _, + _, _, _, _, _, _, + 2018, 4, 15, 17, 59, 8.96092, + 2018, 4, 15, 17, 57, 28.47058, + 2018, 4, 15, 0, 22, 18.77182, + 2018, 4, 15, 22, 43, 5.836905, + 2018, 4, 15, 22, 42, 7.342505, + 2018, 4, 15, 21, 1, 4.91956, + 2018, 4, 15, 20, 58, 39.43348, + 2018, 4, 15, 19, 38, 54.88971, + _, _, _, _, _, _, + 2018, 4, 15, 0, 25, 57.75103, + _, _, _, _, _, _, + _, _, _, _, _, _, + _, _, _, _, _, _, + 2018, 4, 15, 11, 3, 9.407737, + 2018, 4, 15, 11, 21, 51.98771, + 2018, 4, 15, 14, 17, 33.32495, + 2018, 4, 15, 14, 14, 22.84327, + _, _, _, _, _, _, + _, _, _, _, _, _, + _, _, _, _, _, _, + _, _, _, _, _, _, + 2018, 4, 15, 14, 24, 8.474729, + 2018, 4, 15, 15, 21, 23.49041, + 2018, 4, 15, 15, 17, 42.32409, + 2018, 4, 15, 15, 53, 38.77679, + _, _, _, _, _, _, + _, _, _, _, _, _, + _, _, _, _, _, _, + _, _, _, _, _, _, + 2018, 4, 15, 14, 44, 46.54345, + 2018, 4, 15, 9, 8, 10.83098, + _, _, _, _, _, _, + 2018, 4, 15, 0, 34, 26.20252, + 2018, 4, 15, 0, 32, 32.21346, + 2018, 4, 15, 12, 51, 42.2143, + _, _, _, _, _, _, + _, _, _, _, _, _, + 2018, 4, 15, 15, 5, 11.11222, + _, _, _, _, _, _, + _, _, _, _, _, _, + 2018, 4, 15, 9, 36, 8.19276, + 2018, 4, 15, 9, 35, 8.198488, + 2018, 4, 15, 12, 48, 33.2322, + 2018, 4, 15, 14, 27, 37.16796, + 2018, 4, 15, 14, 26, 19.1754, + 2018, 4, 15, 14, 3, 8.123897, + _, _, _, _, _, _, + _, _, _, _, _, _, + _, _, _, _, _, _, + _, _, _, _, _, _, + _, _, _, _, _, _, + _, _, _, _, _, _, + _, _, _, _, _, _, + _, _, _, _, _, _, + _, _, _, _, _, _ ; +} diff --git a/test/soca/testdata/icec_amsr2_south_2.cdl b/test/soca/testdata/icec_amsr2_south_2.cdl new file mode 100644 index 000000000..af5a60251 --- /dev/null +++ b/test/soca/testdata/icec_amsr2_south_2.cdl @@ -0,0 +1,306 @@ +netcdf icec_amsr2_south_2 { +dimensions: + Number_of_X_Dimension = 8 ; + Number_of_Y_Dimension = 8 ; + QM = 1 ; + Time_Dimension = 6 ; +variables: + short Across_X_Dimension(Number_of_X_Dimension) ; + Across_X_Dimension:long_name = " Across X Dimension" ; + Across_X_Dimension:coordinates = "" ; + Across_X_Dimension:units = "1" ; + Across_X_Dimension:_FillValue = -9999s ; + short Along_Y_Dimension(Number_of_Y_Dimension) ; + Along_Y_Dimension:long_name = " Along Y Dimension" ; + Along_Y_Dimension:coordinates = "" ; + Along_Y_Dimension:units = "1" ; + Along_Y_Dimension:_FillValue = -9999s ; + int Bootstrap_Ice_Concentration(Number_of_Y_Dimension, Number_of_X_Dimension) ; + Bootstrap_Ice_Concentration:long_name = "Bootstrap Ice Concentration (Fraction)" ; + Bootstrap_Ice_Concentration:units = "1" ; + Bootstrap_Ice_Concentration:coordinates = "Latitude Longitude" ; + Bootstrap_Ice_Concentration:_FillValue = -9999 ; + short Flags(Number_of_Y_Dimension, Number_of_X_Dimension) ; + Flags:long_name = "Quality Flags" ; + Flags:units = "1" ; + Flags:coordinate = "Latitude Longitude" ; + Flags:_FillValue = -9999s ; + Flags:comment = "0: good, 4: SST missing, 8: Weather, 16: spillover, 32: filled, 64: NT2 Missing" ; + int Latency(Number_of_Y_Dimension, Number_of_X_Dimension) ; + Latency:long_name = "Time in seconds from most current time" ; + Latency:units = "seconds" ; + Latency:coordinate = "Latitude Longitude" ; + Latency:_FillValue = -9999 ; + float Latitude(Number_of_Y_Dimension, Number_of_X_Dimension) ; + Latitude:long_name = "Latitude for EASE-2 grid" ; + Latitude:units = "degrees_north" ; + Latitude:valid_range = -90.f, 90.f ; + Latitude:_FillValue = -9999.f ; + Latitude:standard_name = "latitude" ; + float Longitude(Number_of_Y_Dimension, Number_of_X_Dimension) ; + Longitude:long_name = "Longitude for EASE-2 grid" ; + Longitude:units = "degrees_east" ; + Longitude:valid_range = -180.f, 180.f ; + Longitude:_FillValue = -9999.f ; + Longitude:standard_name = "longitude" ; + int NASA_Team_2_Ice_Concentration(Number_of_Y_Dimension, Number_of_X_Dimension) ; + NASA_Team_2_Ice_Concentration:long_name = "NASA Team 2 Ice Concentration (Fraction)" ; + NASA_Team_2_Ice_Concentration:units = "1" ; + NASA_Team_2_Ice_Concentration:coordinates = "Latitude Longitude" ; + NASA_Team_2_Ice_Concentration:_FillValue = -9999 ; + int NASA_Team_2_Multiyear_Ice(Number_of_Y_Dimension, Number_of_X_Dimension) ; + NASA_Team_2_Multiyear_Ice:long_name = "NASA Team 2 Multiyear Ice (Fraction)" ; + NASA_Team_2_Multiyear_Ice:units = "1" ; + NASA_Team_2_Multiyear_Ice:coordinates = "Latitude Longitude" ; + NASA_Team_2_Multiyear_Ice:_FillValue = -9999 ; + int NT2_minus_Bootstrap(Number_of_Y_Dimension, Number_of_X_Dimension) ; + NT2_minus_Bootstrap:long_name = "NT2 minus Bootstrap (Fraction)" ; + NT2_minus_Bootstrap:units = "1" ; + NT2_minus_Bootstrap:coordinates = "Latitude Longitude" ; + NT2_minus_Bootstrap:_FillValue = -9999 ; + float QM_Num_Grid_Range_25(QM) ; + QM_Num_Grid_Range_25:long_name = "Quality Monitoring: Number of Grid Boxes with Range > 25%" ; + QM_Num_Grid_Range_25:units = "1" ; + QM_Num_Grid_Range_25:_FillValue = -9999.f ; + float QM_Num_Grid_Range_50(QM) ; + QM_Num_Grid_Range_50:long_name = "Quality Monitoring: Number of Grid Boxes with Range > 50%" ; + QM_Num_Grid_Range_50:units = "1" ; + QM_Num_Grid_Range_50:_FillValue = -9999.f ; + float QM_Total_Ice(QM) ; + QM_Total_Ice:long_name = "Quality Monitoring: Total Ice Extent" ; + QM_Total_Ice:units = "1" ; + QM_Total_Ice:_FillValue = -9999.f ; + float QM_Total_Pixels(QM) ; + QM_Total_Pixels:long_name = "Quality Monitoring: Total Number of Pixels" ; + QM_Total_Pixels:units = "1" ; + QM_Total_Pixels:_FillValue = -9999.f ; + int Range_of_Ice_Concentration(Number_of_Y_Dimension, Number_of_X_Dimension) ; + Range_of_Ice_Concentration:long_name = "Range of Ice Concentration (Fraction)" ; + Range_of_Ice_Concentration:units = "1" ; + Range_of_Ice_Concentration:coordinates = "Latitude Longitude" ; + Range_of_Ice_Concentration:_FillValue = -9999 ; + float Scan_Time(Number_of_Y_Dimension, Number_of_X_Dimension, Time_Dimension) ; + Scan_Time:long_name = "Scan line Start Time 6-D for (YYYY, MM, DD, HH, MM, SS) in GMT" ; + Scan_Time:units = "1" ; + Scan_Time:_FillValue = -9999.f ; + +// global attributes: + :Conventions = "CF-1.5" ; + :Metadata_Conventions = "CF-1.5, Unidata Datasset Discovery v1.0" ; + :standard_name_vocabulary = "CF Standard Name Table (version 17, 24 March 2011)" ; + :project = "NPP Data Exploitation: NOAA GCOM-W1 AMSR2" ; + :title = "AMSR2_SEAICE_SH" ; + :summary = "GCOM Seaice Southern Hemisphere Products" ; + :institution = "DOC/NOAA/NESDIS/OSPO > Office of Satellite and Product Operations, NESDIS, NOAA, U.S. Department of Commerce" ; + :naming_authority = "gov.noaa.nesdis.nde" ; + :platform_name = "GCOM-W1" ; + :instrument_name = "AMSR2" ; + :creator_name = "DOC/NOAA/NESDIS/STAR > IOSSPDT Algorithm Team, Center for Satellite Applications and Research, NESDIS, NOAA, U.S. Department of Commerce" ; + :creator_email = "espcoperations@noaa.gov" ; + :creator_url = "http://www.star.nesdis.noaa.gov" ; + :publisher_name = "DOC/NOAA/NESDIS/NDE > NPP Data Exploitation, Center for Satellite Applications and Research, NESDIS, NOAA, U.S. Department of Commerce" ; + :publisher_email = "espcoperations@noaa.gov" ; + :publisher_url = "http://www.ospo.noaa.gov/" ; + :references = "Contact the OSPO PAL to request the ATBD." ; + :processing_level = "NOAA Level 2 data" ; + :cdm_data_type = "Grid" ; + :geospatial_lat_min = -89.9367f ; + :geospatial_lat_max = -34.6223f ; + :geospatial_lon_min = -179.932f ; + :geospatial_lon_max = 179.932f ; + :geospatial_lat_units = "degrees_north" ; + :geospatial_lon_units = "degrees_east" ; + :date_created = "2021-07-01T03:40:15Z" ; + :id = "d4079f1c-3c57-46b3-ac1d-d650937aebde" ; + :Metadata_Link = "AMSR2-SEAICE-SH_v2r2_GW1_s202107010120180_e202107010259170_c202107010340150.nc" ; + :history = "Mon Sep 25 15:42:01 2023: ncks -d Number_of_X_Dimension,100,800,100 -d Number_of_Y_Dimension,100,800,100 /scratch1/NCEPDEV/stmp4/Shastri.Paturi/forAndrew/gdas.20210701/00/icec/AMSR2-SEAICE-SH_v2r2_GW1_s202107010120180_e202107010259170_c202107010340150.nc icec_amsr2_south_2.nc\nCreated by GAASP Version 2.0, Release 2.0" ; + :source = "GAASP-L1R_v2r2_GW1_s202107010120180_e202107010259170_c202107010336520.h5, GAASP-L1R_v2r2_GW1_s202106302341190_e202107010120160_c202107010159060.h5, GAASP-L1R_v2r2_GW1_s202106302202190_e202106302341170_c202107010019490.h5, GAASP-L1R_v2r2_GW1_s202106302020180_e202106302202180_c202106302237500.h5, GAASP-L1R_v2r2_GW1_s202106301838190_e202106302020170_c202106302057270.h5, GAASP-L1R_v2r2_GW1_s202106301659190_e202106301838170_c202106301916000.h5, GAASP-L1R_v2r2_GW1_s202106301520170_e202106301659180_c202106301736190.h5, GAASP-L1R_v2r2_GW1_s202106301341160_e202106301520150_c202106301558020.h5, GAASP-L1R_v2r2_GW1_s202106301205160_e202106301341140_c202106301416190.h5, GAASP-L1R_v2r2_GW1_s202106301026170_e202106301205150_c202106301238160.h5, GAASP-L1R_v2r2_GW1_s202106300847160_e202106301026160_c202106301101410.h5, GAASP-L1R_v2r2_GW1_s202106300711170_e202106300847150_c202106300921380.h5, GAASP-L1R_v2r2_GW1_s202106300532170_e202106300711150_c202106300744420.h5, GAASP-L1R_v2r2_GW1_s202106300356160_e202106300532160_c202106300607150.h5, GAASP-L1R_v2r2_GW1_s202106300217170_e202106300356150_c202106300428400.h5, GAASP-L1R_v2r2_GW1_s202106300038170_e202106300217150_c202106300252420.h5" ; + :production_site = "NSOF" ; + :production_environment = "OE" ; + :time_coverage_start = "2021-07-01T01:20:18.440Z" ; + :time_coverage_end = "2021-07-01T02:59:17.876Z" ; + :start_orbit_number = 48514 ; + :end_orbit_number = 48515 ; + :ascend_descend_data_flag = 2 ; + :day_night_data_flag = 2 ; + :NCO = "netCDF Operators version 5.0.6 (Homepage = http://nco.sf.net, Code = http://github.com/nco/nco)" ; +data: + + Across_X_Dimension = 101, 201, 301, 401, 501, 601, 701, 801 ; + + Along_Y_Dimension = 101, 201, 301, 401, 501, 601, 701, 801 ; + + Bootstrap_Ice_Concentration = + 0, 0, 0, 100, 94, 0, 0, 0, + 0, 94, 100, 82, 95, 84, 0, 0, + 0, _, 99, _, _, _, 97, 0, + 0, 100, _, _, _, _, 100, 0, + 0, 99, _, _, _, _, 95, 0, + 0, 0, 94, 96, _, _, 0, 0, + 0, 0, 0, 0, 91, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0 ; + + Flags = + 4, 8, 8, 0, 0, 8, 8, 4, + 8, 0, 0, 0, 0, 0, 8, 8, + 8, 120, 0, 120, 120, 120, 0, 8, + 8, 0, 120, 120, 120, 120, 0, 8, + 8, 0, 120, 120, 120, 120, 0, 8, + 4, 8, 0, 0, 120, 120, 8, 8, + 4, 8, 8, 8, 0, 8, 8, 4, + 4, 4, 4, 4, 4, 4, 4, 4 ; + + Latency = + _, 636, 61, 159, 159, 259, 261, _, + 537, 540, 58, 156, 256, 257, 358, 360, + 440, 2000, 56, 2000, 2000, 2000, 356, 457, + 341, 344, 2000, 2000, 2000, 2000, 455, 457, + 341, 245, 2000, 2000, 2000, 2000, 554, 1131, + _, 144, 48, 48, 2000, 2000, 654, 557, + _, 143, 45, 46, 751, 752, 656, _, + _, _, _, _, _, _, _, _ ; + + Latitude = + -48.58855, -54.69889, -59.05717, -61.01042, -60.13906, -56.64225, + -51.14237, -44.22784, + -54.69889, -61.90622, -67.46837, -70.16405, -68.94062, -64.3263, -57.65202, + -49.79427, + -59.05717, -67.46837, -74.82165, -79.14207, -77.07041, -70.51806, + -62.41702, -53.64244, + -61.01042, -70.16405, -79.14207, -87.5308, -82.57877, -73.68634, -64.60285, + -55.3238, + -60.13906, -68.94062, -77.07041, -82.57877, -79.79292, -72.22871, + -63.62301, -54.57748, + -56.64225, -64.3263, -70.51806, -73.68634, -72.22871, -66.97952, -59.76062, + -51.52504, + -51.14237, -57.65202, -62.41702, -64.60285, -63.62301, -59.76062, + -53.84447, -46.57473, + -44.22784, -49.79427, -53.64244, -55.3238, -54.57748, -51.52504, -46.57473, + -40.16538 ; + + Longitude = + -45, -34.48949, -20.5069, -3.492593, 14.14168, 29.46407, 41.28101, 49.98034, + -55.51051, -45, -28.56472, -5.07673, 20.14014, 39.43126, 51.95571, 60.02053, + -69.4931, -61.43528, -45, -9.267837, 33.96579, 56.49345, 66.92478, 72.56456, + -86.50741, -84.92327, -80.73216, -45, 76.38319, 83.83407, 86.02327, 87.06625, + -104.1417, -110.1401, -123.9658, -166.3832, 135, 114.0361, 106.0128, + 101.9456, + -119.4641, -129.4313, -146.4935, -173.8341, 155.9639, 135, 122.761, 115.3785, + -131.281, -141.9557, -156.9248, -176.0233, 163.9872, 147.239, 135, 126.3972, + -139.9803, -150.0205, -162.5646, -177.0663, 168.0544, 154.6215, 143.6028, + 135 ; + + NASA_Team_2_Ice_Concentration = + 0, 0, 0, 100, 100, 0, 0, 0, + 0, 100, 100, 100, 100, 99, 0, 0, + 0, _, 100, _, _, _, 100, 0, + 0, 99, _, _, _, _, 100, 0, + 0, 100, _, _, _, _, 100, 0, + 0, 0, 100, 100, _, _, 0, 0, + 0, 0, 0, 0, 100, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0 ; + + NASA_Team_2_Multiyear_Ice = + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, _, 0, _, _, _, 0, 0, + 0, 0, _, _, _, _, 0, 0, + 0, 57, _, _, _, _, 0, 0, + 0, 0, 0, 0, _, _, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0 ; + + NT2_minus_Bootstrap = + 0, 0, 0, 0, 6, 0, 0, 0, + 0, 6, 0, 18, 5, 15, 0, 0, + 0, _, 1, _, _, _, 3, 0, + 0, 0, _, _, _, _, 0, 0, + 0, 1, _, _, _, _, 5, 0, + 0, 0, 6, 4, _, _, 0, 0, + 0, 0, 0, 0, 9, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0 ; + + QM_Num_Grid_Range_25 = 3269 ; + + QM_Num_Grid_Range_50 = 775 ; + + QM_Total_Ice = 148706 ; + + QM_Total_Pixels = 705600 ; + + Range_of_Ice_Concentration = + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1, 2, 0, 0, 1, 0, 0, + 0, _, 0, _, _, _, 0, 0, + 0, 1, _, _, _, _, 0, 0, + 0, 4, _, _, _, _, 0, 0, + 0, 0, 0, 0, _, _, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0 ; + + Scan_Time = + _, _, _, _, _, _, + 2018, 4, 15, 10, 22, 42.51182, + 2018, 4, 15, 11, 57, 58.72683, + 2018, 4, 15, 10, 20, 8.284265, + 2018, 4, 15, 10, 19, 36.78727, + 2018, 4, 15, 12, 39, 34.35703, + 2018, 4, 15, 12, 38, 11.86481, + _, _, _, _, _, _, + 2018, 4, 15, 11, 1, 22.44809, + 2018, 4, 15, 10, 59, 8.96092, + 2018, 4, 15, 12, 0, 28.71251, + 2018, 4, 15, 10, 22, 18.77182, + 2018, 4, 15, 12, 43, 5.836905, + 2018, 4, 15, 12, 42, 7.342505, + 2018, 4, 15, 11, 1, 4.91956, + 2018, 4, 15, 10, 58, 39.43348, + 2018, 4, 15, 13, 38, 54.88971, + _, _, _, _, _, _, + 2018, 4, 15, 2, 2, 57.19851, + _, _, _, _, _, _, + _, _, _, _, _, _, + _, _, _, _, _, _, + 2018, 4, 15, 21, 3, 9.407737, + 2018, 4, 15, 19, 21, 51.98771, + 2018, 4, 15, 21, 17, 33.32495, + 2018, 4, 15, 21, 14, 22.84327, + _, _, _, _, _, _, + _, _, _, _, _, _, + _, _, _, _, _, _, + _, _, _, _, _, _, + 2018, 4, 15, 19, 24, 8.474729, + 2018, 4, 15, 19, 21, 23.49041, + 2018, 4, 15, 21, 17, 42.32409, + 2018, 4, 15, 22, 53, 38.77679, + _, _, _, _, _, _, + _, _, _, _, _, _, + _, _, _, _, _, _, + _, _, _, _, _, _, + 2018, 4, 15, 12, 44, 46.54345, + 2018, 4, 15, 8, 8, 10.83098, + _, _, _, _, _, _, + 2018, 4, 15, 0, 34, 26.20252, + 2018, 4, 15, 2, 10, 45.15401, + 2018, 4, 15, 2, 10, 49.65358, + _, _, _, _, _, _, + _, _, _, _, _, _, + 2018, 4, 15, 14, 5, 11.11222, + 2018, 4, 15, 14, 41, 39.06126, + _, _, _, _, _, _, + 2018, 4, 15, 0, 36, 8.19276, + 2018, 4, 15, 2, 14, 3.135001, + 2018, 4, 15, 2, 13, 6.140425, + 2018, 4, 15, 14, 27, 37.16796, + 2018, 4, 15, 14, 26, 19.1754, + 2018, 4, 15, 14, 3, 8.123897, + _, _, _, _, _, _, + _, _, _, _, _, _, + _, _, _, _, _, _, + _, _, _, _, _, _, + _, _, _, _, _, _, + _, _, _, _, _, _, + _, _, _, _, _, _, + _, _, _, _, _, _, + _, _, _, _, _, _ ; +} diff --git a/test/soca/testdata/rads_adt_3a_2018105.cdl b/test/soca/testdata/rads_adt_3a_2018105.cdl new file mode 100644 index 000000000..3e08dd0f1 --- /dev/null +++ b/test/soca/testdata/rads_adt_3a_2018105.cdl @@ -0,0 +1,110 @@ +netcdf rads_adt_3a_2021181 { +dimensions: + time = UNLIMITED ; // (11 currently) +variables: + int adt_egm2008(time) ; + adt_egm2008:_FillValue = 2147483647 ; + adt_egm2008:long_name = "absolute dynamic topography (EGM2008)" ; + adt_egm2008:standard_name = "absolute_dynamic_topography_egm2008" ; + adt_egm2008:units = "m" ; + adt_egm2008:scale_factor = 0.0001 ; + adt_egm2008:coordinates = "lon lat" ; + int adt_xgm2016(time) ; + adt_xgm2016:_FillValue = 2147483647 ; + adt_xgm2016:long_name = "absolute dynamic topography (XGM2016)" ; + adt_xgm2016:standard_name = "absolute_dynamic_topography_xgm2016" ; + adt_xgm2016:units = "m" ; + adt_xgm2016:scale_factor = 0.0001 ; + adt_xgm2016:coordinates = "lon lat" ; + int cycle(time) ; + cycle:_FillValue = 2147483647 ; + cycle:long_name = "cycle number" ; + cycle:field = 9905s ; + int lat(time) ; + lat:_FillValue = 2147483647 ; + lat:long_name = "latitude" ; + lat:standard_name = "latitude" ; + lat:units = "degrees_north" ; + lat:scale_factor = 1.e-06 ; + lat:field = 201s ; + lat:comment = "Positive latitude is North latitude, negative latitude is South latitude" ; + int lon(time) ; + lon:_FillValue = 2147483647 ; + lon:long_name = "longitude" ; + lon:standard_name = "longitude" ; + lon:units = "degrees_east" ; + lon:scale_factor = 1.e-06 ; + lon:field = 301s ; + lon:comment = "East longitude relative to Greenwich meridian" ; + int pass(time) ; + pass:_FillValue = 2147483647 ; + pass:long_name = "pass number" ; + pass:field = 9906s ; + short sla(time) ; + sla:_FillValue = 32767s ; + sla:long_name = "sea level anomaly" ; + sla:standard_name = "sea_surface_height_above_sea_level" ; + sla:units = "m" ; + sla:quality_flag = "swh sig0 range_rms range_numval flags swh_rms sig0_rms" ; + sla:scale_factor = 0.0001 ; + sla:coordinates = "lon lat" ; + sla:field = 0s ; + sla:comment = "Sea level determined from satellite altitude - range - all altimetric corrections" ; + double time_dtg(time) ; + time_dtg:long_name = "time_dtg" ; + time_dtg:standard_name = "time_dtg" ; + time_dtg:units = "yyyymmddhhmmss" ; + time_dtg:coordinates = "lon lat" ; + time_dtg:comment = "UTC time formatted as yyyymmddhhmmss" ; + double time_mjd(time) ; + time_mjd:long_name = "Modified Julian Days" ; + time_mjd:standard_name = "time" ; + time_mjd:units = "days since 1858-11-17 00:00:00 UTC" ; + time_mjd:field = 105s ; + time_mjd:comment = "UTC time of measurement expressed in Modified Julian Days" ; + +// global attributes: + :Conventions = "CF-1.7" ; + :title = "RADS 4 pass file" ; + :institution = "EUMETSAT / NOAA / TU Delft" ; + :source = "radar altimeter" ; + :references = "RADS Data Manual, Version 4.2 or later" ; + :featureType = "trajectory" ; + :ellipsoid = "TOPEX" ; + :ellipsoid_axis = 6378136.3 ; + :ellipsoid_flattening = 0.00335281317789691 ; + :filename = "rads_adt_3a_2021181.nc" ; + :mission_name = "SNTNL-3A" ; + :mission_phase = "a" ; + :log01 = "2021-07-01 | /Users/rads/bin/rads2nc --ymd=20210630000000,20210701000000 -C1,1000 -S3a -Vsla,adt_egm2008,adt_xgm2016,time_mjd,time_dtg,lon,lat,cycle,pass -X/Users/rads/cron/xgm2016 -X/Users/rads/cron/adt -X/Users/rads/cron/time_dtg -o/Users/rads/adt/2021/181/rads_adt_3a_2021181.nc: RAW data from" ; + :history = "Mon Sep 25 17:01:30 2023: ncks -d time,0,10 rads_adt_3a_2021181.nc rads_adt_3a_2021181.ncn\n", + "2021-07-01 21:14:30 : /Users/rads/bin/rads2nc --ymd=20210630000000,20210701000000 -C1,1000 -S3a -Vsla,adt_egm2008,adt_xgm2016,time_mjd,time_dtg,lon,lat,cycle,pass -X/Users/rads/cron/xgm2016 -X/Users/rads/cron/adt -X/Users/rads/cron/time_dtg -o/Users/rads/adt/2021/181/rads_adt_3a_2021181.nc" ; + :NCO = "netCDF Operators version 5.0.6 (Homepage = http://nco.sf.net, Code = http://github.com/nco/nco)" ; +data: + + adt_egm2008 = 1674, 2176, 1671, 2320, 2011, 2349, 2139, 2174, 2462, 2441, + 2487 ; + + adt_xgm2016 = 1621, 2300, 1938, 2652, 2093, 2246, 1948, 1976, 2406, 2469, + 2649 ; + + cycle = 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73 ; + + lat = -43973134, -43915104, -43857071, -43799035, -43740996, -43682953, + -43624907, -43566858, -43508805, -43450749, -43392690 ; + + lon = -21550109, -21571346, -21592549, -21613720, -21634859, -21655964, + -21677037, -21698078, -21719086, -21740062, -21761007 ; + + pass = 517, 517, 517, 517, 517, 517, 517, 517, 517, 517, 517 ; + + sla = 75, 523, -338, 671, 104, 513, 128, 108, 424, 322, 497 ; + + time_dtg = 20180415000000, 20180415000001, 20180415000002, 20180415000003, + 20180415000004, 20180415000005, 20180415000006, 20180415000007, + 20180415000008, 20180415000009, 20180415000010 ; + + time_mjd = 58223.5, 58223.5, 58223.5, 58223.5, + 58223.5, 58223.5, 58223.5, 58223.5, + 58223.5, 58223.5, 58223.5 ; +} diff --git a/test/soca/testdata/rads_adt_3b_2018105.cdl b/test/soca/testdata/rads_adt_3b_2018105.cdl new file mode 100644 index 000000000..c43aef2f2 --- /dev/null +++ b/test/soca/testdata/rads_adt_3b_2018105.cdl @@ -0,0 +1,110 @@ +netcdf rads_adt_3b_2021181 { +dimensions: + time = UNLIMITED ; // (11 currently) +variables: + int adt_egm2008(time) ; + adt_egm2008:_FillValue = 2147483647 ; + adt_egm2008:long_name = "absolute dynamic topography (EGM2008)" ; + adt_egm2008:standard_name = "absolute_dynamic_topography_egm2008" ; + adt_egm2008:units = "m" ; + adt_egm2008:scale_factor = 0.0001 ; + adt_egm2008:coordinates = "lon lat" ; + int adt_xgm2016(time) ; + adt_xgm2016:_FillValue = 2147483647 ; + adt_xgm2016:long_name = "absolute dynamic topography (XGM2016)" ; + adt_xgm2016:standard_name = "absolute_dynamic_topography_xgm2016" ; + adt_xgm2016:units = "m" ; + adt_xgm2016:scale_factor = 0.0001 ; + adt_xgm2016:coordinates = "lon lat" ; + int cycle(time) ; + cycle:_FillValue = 2147483647 ; + cycle:long_name = "cycle number" ; + cycle:field = 9905s ; + int lat(time) ; + lat:_FillValue = 2147483647 ; + lat:long_name = "latitude" ; + lat:standard_name = "latitude" ; + lat:units = "degrees_north" ; + lat:scale_factor = 1.e-06 ; + lat:field = 201s ; + lat:comment = "Positive latitude is North latitude, negative latitude is South latitude" ; + int lon(time) ; + lon:_FillValue = 2147483647 ; + lon:long_name = "longitude" ; + lon:standard_name = "longitude" ; + lon:units = "degrees_east" ; + lon:scale_factor = 1.e-06 ; + lon:field = 301s ; + lon:comment = "East longitude relative to Greenwich meridian" ; + int pass(time) ; + pass:_FillValue = 2147483647 ; + pass:long_name = "pass number" ; + pass:field = 9906s ; + short sla(time) ; + sla:_FillValue = 32767s ; + sla:long_name = "sea level anomaly" ; + sla:standard_name = "sea_surface_height_above_sea_level" ; + sla:units = "m" ; + sla:quality_flag = "swh sig0 range_rms range_numval flags swh_rms sig0_rms" ; + sla:scale_factor = 0.0001 ; + sla:coordinates = "lon lat" ; + sla:field = 0s ; + sla:comment = "Sea level determined from satellite altitude - range - all altimetric corrections" ; + double time_dtg(time) ; + time_dtg:long_name = "time_dtg" ; + time_dtg:standard_name = "time_dtg" ; + time_dtg:units = "yyyymmddhhmmss" ; + time_dtg:coordinates = "lon lat" ; + time_dtg:comment = "UTC time formatted as yyyymmddhhmmss" ; + double time_mjd(time) ; + time_mjd:long_name = "Modified Julian Days" ; + time_mjd:standard_name = "time" ; + time_mjd:units = "days since 1858-11-17 00:00:00 UTC" ; + time_mjd:field = 105s ; + time_mjd:comment = "UTC time of measurement expressed in Modified Julian Days" ; + +// global attributes: + :Conventions = "CF-1.7" ; + :title = "RADS 4 pass file" ; + :institution = "EUMETSAT / NOAA / TU Delft" ; + :source = "radar altimeter" ; + :references = "RADS Data Manual, Version 4.2 or later" ; + :featureType = "trajectory" ; + :ellipsoid = "TOPEX" ; + :ellipsoid_axis = 6378136.3 ; + :ellipsoid_flattening = 0.00335281317789691 ; + :filename = "rads_adt_3b_2021181.nc" ; + :mission_name = "SNTNL-3B" ; + :mission_phase = "b" ; + :log01 = "2021-07-01 | /Users/rads/bin/rads2nc --ymd=20210630000000,20210701000000 -S3b -Vsla,adt_egm2008,adt_xgm2016,time_mjd,time_dtg,lon,lat,cycle,pass -X/Users/rads/cron/xgm2016 -X/Users/rads/cron/adt -X/Users/rads/cron/time_dtg -o/Users/rads/adt/2021/181/rads_adt_3b_2021181.nc: RAW data from" ; + :history = "Mon Sep 25 17:01:31 2023: ncks -d time,0,10 rads_adt_3b_2021181.nc rads_adt_3b_2021181.ncn\n", + "2021-07-01 21:31:13 : /Users/rads/bin/rads2nc --ymd=20210630000000,20210701000000 -S3b -Vsla,adt_egm2008,adt_xgm2016,time_mjd,time_dtg,lon,lat,cycle,pass -X/Users/rads/cron/xgm2016 -X/Users/rads/cron/adt -X/Users/rads/cron/time_dtg -o/Users/rads/adt/2021/181/rads_adt_3b_2021181.nc" ; + :NCO = "netCDF Operators version 5.0.6 (Homepage = http://nco.sf.net, Code = http://github.com/nco/nco)" ; +data: + + adt_egm2008 = 6505, 7307, 6026, 5871, 5561, 5246, 4981, 4661, 4391, 4409, + 4283 ; + + adt_xgm2016 = 3136, 4247, 4498, 4515, 4579, 4741, 4731, 4351, 4016, 3860, + 3887 ; + + cycle = 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54 ; + + lat = 59732735, 59505896, 58767628, 58710774, 58653911, 58597040, 58540159, + 58483270, 58426372, 58369465, 58312550 ; + + lon = 163417447, 163262248, 162770394, 162733328, 162696369, 162659516, + 162622770, 162586128, 162549591, 162513159, 162476830 ; + + pass = 232, 232, 232, 232, 232, 232, 232, 232, 232, 232, 232 ; + + sla = 4578, 5072, 1437, 756, 254, 91, -2, -170, -484, -313, -244 ; + + time_dtg = 20180415000629, 20180415000633, 20180415000646, 20180415000647, + 20180415000648, 20180415000649, 20180415000650, 20180415000651, + 20180415000652, 20180415000653, 20180415000654 ; + + time_mjd = 58223.5, 58223.5, 58223.5, + 58223.5, 58223.5, 58223.5, 58223.5, + 58223.5, 58223.5, 58223.5, 58223.5 ; +} diff --git a/ush/ufsda/soca_utils.py b/ush/ufsda/soca_utils.py deleted file mode 100644 index 83bfb9d7b..000000000 --- a/ush/ufsda/soca_utils.py +++ /dev/null @@ -1,118 +0,0 @@ -import xarray -import glob -import numpy as np -import sys -import os -import logging -import shutil - -# TODO: We might want to revisit this in the future -# Try to resolve the location of pyioda, assuming there are only 2 places where this -# script can exist (build/ush/ufsda or /ush/ufsda) -from pathlib import Path -jedilib = Path(os.path.join(Path(__file__).parent.absolute(), '../..', 'lib')) -if not jedilib.is_dir(): - jedilib = Path(os.path.join(Path(__file__).parent.absolute(), '../../build', 'lib')) -pyver = 'python3.'+str(sys.version_info[1]) -pyioda_lib = Path(os.path.join(jedilib, pyver, 'pyioda')).resolve() -pyiodaconv_lib = Path(os.path.join(jedilib, 'pyiodaconv')).resolve() -sys.path.append(str(pyioda_lib)) -sys.path.append(str(pyiodaconv_lib)) -import pyiodaconv.ioda_conv_engines as iconv -from pyiodaconv.orddicts import DefaultOrderedDict - -__all__ = ['concatenate_ioda'] - -logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', - level=logging.INFO, datefmt='%Y-%m-%d %H:%M:%S') - -marine_obs_names = ['seaSurfaceTemperature', - 'absoluteDynamicTopography', - 'seaIceFraction', - 'seaIceFreeboard', - 'waterTemperature', - 'salinity', - 'seaSurfaceSalinity'] - -iodatypes = {"ObsError": 'float32', - "ObsValue": 'float32', - "PreQC": 'int32'} - - -def obs_specs(iodafname): - with xarray.open_dataset(iodafname, group='MetaData', decode_cf=False) as ds: - timeref = ds['dateTime'].attrs['units'] - - with xarray.open_dataset(iodafname, group='ObsValue') as ds: - for varname in ds.variables: - if varname in marine_obs_names: - logging.info(f"Found {varname} in {iodafname}.") - var = ds[varname] - return varname, var.attrs['units'], timeref - - raise Exception(f"No known obs type in {iodafname}.") - - -def concatenate_ioda(iodafname, wildcard=".*", output_suffix="", clean=False): - flist = glob.glob(f"{iodafname}{wildcard}") - flist.sort() - nfiles = len(flist) - if nfiles == 0: - logging.info(f"No files to concatenate.") - return - - if len(flist) == 1: - logging.info(f"Only file is {flist[0]}, rename to {iodafname}. No need to concatenate.") - shutil.move(flist[0], iodafname+output_suffix) - return - - logging.info(f"Concatenating {nfiles} files from globbing {iodafname}.*") - - # Identify variable and unit - obsvarname, units, timeref = obs_specs(flist[0]) - - # Concatenate stuff outside of groups (Location dimensions and variables) - ds = xarray.concat([xarray.open_dataset(f) for f in flist], dim='Location') - - # Concatenate all groups except MetaData - outdata = {} - for group in ["ObsError", "ObsValue", "PreQC"]: - ds = xarray.concat([xarray.open_dataset(f, group=group) for f in flist], dim='Location') - outdata[(obsvarname, group)] = ds[obsvarname].astype(iodatypes[group]) - - # Concatenate metadata - ds = xarray.concat([xarray.open_dataset(f, group="MetaData", decode_cf=False) for f in flist], dim='Location') - for k in list(ds.keys()): - outdata[(k, "MetaData")] = ds[k] - - # Setup the IODA writer - varattrs = DefaultOrderedDict(lambda: DefaultOrderedDict(dict)) - - # obs attributes - varattrs[(obsvarname, "ObsValue")]['units'] = units - varattrs[(obsvarname, "ObsError")]['units'] = units - - # meta data attributes - varattrs[('latitude', 'MetaData')]['units'] = 'degrees_north' - varattrs[('longitude', 'MetaData')]['units'] = 'degrees_east' - varattrs[('dateTime', 'MetaData')]['units'] = timeref - locationkeylist = [("latitude", "float", "degrees_north"), - ("longitude", "float", "degrees_east"), - ("dateTime", "long", timeref)] - - vardims = {'': ['Location']} - nlocs = ds.dims['Location'] - dimdict = {} - dimdict['Location'] = nlocs - globalattrs = {} - - # Write - writer = iconv.IodaWriter(iodafname+output_suffix, locationkeylist, dimdict) - writer.BuildIoda(outdata, vardims, varattrs, globalattrs) - - # Remove files - if clean: - for file in flist: - os.remove(file) - - return diff --git a/ush/ufsda/stage.py b/ush/ufsda/stage.py index 84c83a2ef..de0962a0e 100644 --- a/ush/ufsda/stage.py +++ b/ush/ufsda/stage.py @@ -12,7 +12,6 @@ import glob import numpy as np from wxflow import YAMLFile, parse_yaml, parse_j2yaml -import ufsda.soca_utils __all__ = ['atm_background', 'atm_obs', 'bias_obs', 'background', 'background_ens', 'fv3jedi', 'obs', 'berror', 'gdas_fix', 'gdas_single_cycle'] From cc4c940f98242674a20d8e3950783011387fac6c Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA <26926959+RussTreadon-NOAA@users.noreply.github.com> Date: Thu, 11 Jan 2024 16:09:51 -0500 Subject: [PATCH 4/4] Add Hercules build capability to GDASApp (#774) MSU Hercules is available for use. This PR adds `hercules.lua` to GDASApp `modulefiles/GDAS`. Fixes #773 --------- Co-authored-by: Cory Martin --- build.sh | 2 +- modulefiles/GDAS/hercules.lua | 98 ++++++++++++++++++++++ test/atm/global-workflow/jjob_ens_final.sh | 10 +-- test/atm/global-workflow/jjob_ens_init.sh | 14 ++-- test/atm/global-workflow/jjob_ens_run.sh | 10 +-- test/atm/global-workflow/jjob_var_final.sh | 10 +-- test/atm/global-workflow/jjob_var_init.sh | 18 ++-- test/atm/global-workflow/jjob_var_run.sh | 10 +-- test/soca/gw/CMakeLists.txt | 11 ++- ush/detect_machine.sh | 2 + ush/module-setup.sh | 7 ++ ush/soca/run_jjobs.py | 2 +- 12 files changed, 153 insertions(+), 41 deletions(-) create mode 100644 modulefiles/GDAS/hercules.lua diff --git a/build.sh b/build.sh index 1d80abba8..e1bd96688 100755 --- a/build.sh +++ b/build.sh @@ -70,7 +70,7 @@ while getopts "p:t:c:hvdfa" opt; do done case ${BUILD_TARGET} in - hera | orion) + hera | orion | hercules) echo "Building GDASApp on $BUILD_TARGET" source $dir_root/ush/module-setup.sh module use $dir_root/modulefiles diff --git a/modulefiles/GDAS/hercules.lua b/modulefiles/GDAS/hercules.lua new file mode 100644 index 000000000..e989af14a --- /dev/null +++ b/modulefiles/GDAS/hercules.lua @@ -0,0 +1,98 @@ +help([[ +Load environment for running the GDAS application with Intel compilers and MPI. +]]) + +local pkgName = myModuleName() +local pkgVersion = myModuleVersion() +local pkgNameVer = myModuleFullName() + +prepend_path("MODULEPATH", '/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core') +prepend_path("MODULEPATH", '/work2/noaa/da/python/opt/modulefiles/stack') + +-- below two lines get us access to the spack-stack modules +load("stack-intel/2021.9.0") +load("stack-intel-oneapi-mpi/2021.9.0") +--load("stack-python/3.10.8") +-- JCSDA has 'jedi-fv3-env/unified-dev', but we should load these manually as needed +load("cmake/3.23.1") +load("curl/8.1.2") +load("zlib/1.2.13") +load("git/2.40.0") +--load("pkg-config/0.27.1") +load("hdf5/1.14.0") +load("parallel-netcdf/1.12.2") +load("netcdf-c/4.9.2") +load("nccmp/1.9.0.1") +load("netcdf-fortran/4.6.0") +load("nco/5.0.6") +load("parallelio/2.5.10") +load("wget/1.21.1") +load("boost/1.78") +load("bufr/12.0.1") +load("git-lfs/3.1.2") +load("ecbuild/3.7.2") +load("openjpeg/2.4.0") +load("eccodes/2.27.0") +load("eigen/3.4.0") +load("openblas/0.3.19") +load("eckit/1.24.4") +load("fftw/3.3.10") +load("fckit/0.11.0") +load("fiat/1.2.0") +load("ectrans/1.2.0") +load("atlas/0.35.0") +load("sp/2.3.3") +load("gsl-lite/0.37.0") +load("libjpeg/2.1.0") +load("krb5/1.20.1") +load("libtirpc/1.2.6") +load("hdf/4.2.15") +load("jedi-cmake/1.4.0") +load("libpng/1.6.37") +load("libxt/1.1.5") +load("libxmu/1.1.4") +load("libxpm/3.5.12") +load("libxaw/1.0.13") +load("udunits/2.2.28") +load("ncview/2.1.8") +load("netcdf-cxx4/4.3.1") +load("py-pybind11/2.8.1") +--load("crtm/v2.4_jedi") +load("contrib/0.1") +load("noaatools/3.1") +load("rocoto/1.3.5") + +load("hpc/1.2.0") +unload("python/3.10.8") +unload("python/3.9.2") +load("miniconda3/4.6.14") +load("gdasapp/1.0.0") + +load("boost/1.78.0") +load("bufr/12.0.1") +load("fckit/0.11.0") +load("atlas/0.35.0") +load("py-pybind11/2.8.1") + +-- below is a hack because of cmake finding the wrong python... +setenv("CONDA_PREFIX", "/work2/noaa/da/python/opt/core/miniconda3/4.6.14/envs/gdasapp/") + +setenv("CC","mpiicc") +setenv("FC","mpiifort") +setenv("CXX","mpiicpc") +local mpiexec = '/opt/slurm/bin/srun' +local mpinproc = '-n' +setenv('MPIEXEC_EXEC', mpiexec) +setenv('MPIEXEC_NPROC', mpinproc) + +setenv('R2D2_CONFIG', '/work2/noaa/da/cmartin/GDASApp/R2D2_SHARED/config_orion.yaml') +setenv("CRTM_FIX","/work2/noaa/da/cmartin/GDASApp/fix/crtm/2.4.0") +setenv("GDASAPP_TESTDATA","/work2/noaa/da/cmartin/CI/GDASApp/data") +prepend_path("PATH","/apps/contrib/NCEP/libs/hpc-stack/intel-2018.4/prod_util/1.2.2/bin") + +execute{cmd="ulimit -s unlimited",modeA={"load"}} + +whatis("Name: ".. pkgName) +whatis("Version: ".. pkgVersion) +whatis("Category: GDASApp") +whatis("Description: Load all libraries needed for GDASApp") diff --git a/test/atm/global-workflow/jjob_ens_final.sh b/test/atm/global-workflow/jjob_ens_final.sh index ae1928a18..f0a0a9117 100755 --- a/test/atm/global-workflow/jjob_ens_final.sh +++ b/test/atm/global-workflow/jjob_ens_final.sh @@ -33,11 +33,11 @@ export PYTHONPATH machine=$(echo `grep 'machine=' $EXPDIR/config.base | cut -d"=" -f2` | tr -d '"') # Set NETCDF and UTILROOT variables (used in config.base) -if [ $machine = 'HERA' ]; then +if [[ $machine = 'HERA' ]]; then NETCDF=$( which ncdump ) export NETCDF export UTILROOT="/scratch2/NCEPDEV/ensemble/save/Walter.Kolczynski/hpc-stack/intel-18.0.5.274/prod_util/1.2.2" -elif [ $machine = 'ORION' ]; then +elif [[ $machine = 'ORION' || $machine = 'HERCULES' ]]; then ncdump=$( which ncdump ) NETCDF=$( echo "${ncdump}" | cut -d " " -f 3 ) export NETCDF @@ -45,10 +45,10 @@ elif [ $machine = 'ORION' ]; then fi # Execute j-job -if [ $machine = 'HERA' ]; then +if [[ $machine = 'HERA' ]]; then + sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE +elif [[ $machine = 'ORION' || $machine = 'HERCULES' ]]; then sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE -elif [ $machine = 'ORION' ]; then - sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --partition=orion --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE else ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE fi diff --git a/test/atm/global-workflow/jjob_ens_init.sh b/test/atm/global-workflow/jjob_ens_init.sh index 8966f2b69..e46ea90da 100755 --- a/test/atm/global-workflow/jjob_ens_init.sh +++ b/test/atm/global-workflow/jjob_ens_init.sh @@ -38,11 +38,11 @@ export PYTHONPATH machine=$(echo `grep 'machine=' $EXPDIR/config.base | cut -d"=" -f2` | tr -d '"') # Set NETCDF and UTILROOT variables (used in config.base) -if [ $machine = 'HERA' ]; then +if [[ $machine = 'HERA' ]]; then NETCDF=$( which ncdump ) export NETCDF export UTILROOT="/scratch2/NCEPDEV/ensemble/save/Walter.Kolczynski/hpc-stack/intel-18.0.5.274/prod_util/1.2.2" -elif [ $machine = 'ORION' ]; then +elif [[ $machine = 'ORION' || $machine = 'HERCULES' ]]; then ncdump=$( which ncdump ) NETCDF=$( echo "${ncdump}" | cut -d " " -f 3 ) export NETCDF @@ -69,7 +69,7 @@ dpath=gdas.$PDY/$cyc/obs mkdir -p $COM_OBS flist="amsua_n19.$CDATE.nc4 sondes.$CDATE.nc4" for file in $flist; do - ln -fs $GDASAPP_TESTDATA/lowres/$dpath/${oprefix}.$file $COM_OBS/ + ln -fs $GDASAPP_TESTDATA/lowres/$dpath/${oprefix}.$file $COM_OBS/${oprefix}.$file done # Link radiance bias correction files @@ -77,7 +77,7 @@ dpath=gdas.$gPDY/$gcyc/analysis/atmos mkdir -p $COM_ATMOS_ANALYSIS_PREV flist="amsua_n19.satbias.nc4 amsua_n19.satbias_cov.nc4 amsua_n19.tlapse.txt" for file in $flist; do - ln -fs $GDASAPP_TESTDATA/lowres/$dpath/$gprefix.$file $COM_ATMOS_ANALYSIS_PREV/ + ln -fs $GDASAPP_TESTDATA/lowres/$dpath/$gprefix.$file $COM_ATMOS_ANALYSIS_PREV/$gprefix.$file done # Link member atmospheric background on tiles and atmf006 @@ -104,10 +104,10 @@ for imem in $(seq 1 $NMEM_ENS); do done # Execute j-job -if [ $machine = 'HERA' ]; then +if [[ $machine = 'HERA' ]]; then + sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE +elif [[ $machine = 'ORION' || $machine = 'HERCULES' ]]; then sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE -elif [ $machine = 'ORION' ]; then - sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --partition=orion --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE else ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE fi diff --git a/test/atm/global-workflow/jjob_ens_run.sh b/test/atm/global-workflow/jjob_ens_run.sh index 597afcd43..89bff09fe 100755 --- a/test/atm/global-workflow/jjob_ens_run.sh +++ b/test/atm/global-workflow/jjob_ens_run.sh @@ -36,11 +36,11 @@ export PYTHONPATH machine=$(echo `grep 'machine=' $EXPDIR/config.base | cut -d"=" -f2` | tr -d '"') # Set NETCDF and UTILROOT variables (used in config.base) -if [ $machine = 'HERA' ]; then +if [[ $machine = 'HERA' ]]; then NETCDF=$( which ncdump ) export NETCDF export UTILROOT="/scratch2/NCEPDEV/ensemble/save/Walter.Kolczynski/hpc-stack/intel-18.0.5.274/prod_util/1.2.2" -elif [ $machine = 'ORION' ]; then +elif [[ $machine = 'ORION' || $machine = 'HERCULES' ]]; then ncdump=$( which ncdump ) NETCDF=$( echo "${ncdump}" | cut -d " " -f 3 ) export NETCDF @@ -48,10 +48,10 @@ elif [ $machine = 'ORION' ]; then fi # Execute j-job -if [ $machine = 'HERA' ]; then +if [[ $machine = 'HERA' ]]; then + sbatch --nodes=1 --ntasks=36 --account=$ACCOUNT --qos=batch --time=00:30:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN +elif [[ $machine = 'ORION' || $machine = 'HERCULES' ]]; then sbatch --nodes=1 --ntasks=36 --account=$ACCOUNT --qos=batch --time=00:30:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN -elif [ $machine = 'ORION' ]; then - sbatch --nodes=1 --ntasks=36 --account=$ACCOUNT --qos=batch --partition=orion --time=00:30:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN else ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN fi diff --git a/test/atm/global-workflow/jjob_var_final.sh b/test/atm/global-workflow/jjob_var_final.sh index f9b64afbe..01b3c555f 100755 --- a/test/atm/global-workflow/jjob_var_final.sh +++ b/test/atm/global-workflow/jjob_var_final.sh @@ -33,11 +33,11 @@ export PYTHONPATH machine=$(echo `grep 'machine=' $EXPDIR/config.base | cut -d"=" -f2` | tr -d '"') # Set NETCDF and UTILROOT variables (used in config.base) -if [ $machine = 'HERA' ]; then +if [[ $machine = 'HERA' ]]; then NETCDF=$( which ncdump ) export NETCDF export UTILROOT="/scratch2/NCEPDEV/ensemble/save/Walter.Kolczynski/hpc-stack/intel-18.0.5.274/prod_util/1.2.2" -elif [ $machine = 'ORION' ]; then +elif [[ $machine = 'ORION' || $machine = 'HERCULES' ]]; then ncdump=$( which ncdump ) NETCDF=$( echo "${ncdump}" | cut -d " " -f 3 ) export NETCDF @@ -45,10 +45,10 @@ elif [ $machine = 'ORION' ]; then fi # Execute j-job -if [ $machine = 'HERA' ]; then +if [[ $machine = 'HERA' ]]; then + sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_FINALIZE +elif [[ $machine = 'ORION' || $machine = 'HERCULES' ]]; then sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_FINALIZE -elif [ $machine = 'ORION' ]; then - sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --partition=orion --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_FINALIZE else ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_FINALIZE fi diff --git a/test/atm/global-workflow/jjob_var_init.sh b/test/atm/global-workflow/jjob_var_init.sh index 0d88c2174..6d304e0a8 100755 --- a/test/atm/global-workflow/jjob_var_init.sh +++ b/test/atm/global-workflow/jjob_var_init.sh @@ -38,11 +38,11 @@ export PYTHONPATH machine=$(echo `grep 'machine=' $EXPDIR/config.base | cut -d"=" -f2` | tr -d '"') # Set NETCDF and UTILROOT variables (used in config.base) -if [ $machine = 'HERA' ]; then +if [[ $machine = 'HERA' ]]; then NETCDF=$( which ncdump ) export NETCDF export UTILROOT="/scratch2/NCEPDEV/ensemble/save/Walter.Kolczynski/hpc-stack/intel-18.0.5.274/prod_util/1.2.2" -elif [ $machine = 'ORION' ]; then +elif [[ $machine = 'ORION' || $machine = 'HERCULES' ]]; then ncdump=$( which ncdump ) NETCDF=$( echo "${ncdump}" | cut -d " " -f 3 ) export NETCDF @@ -71,7 +71,7 @@ dpath=gdas.$PDY/$cyc/obs mkdir -p $COM_OBS flist="amsua_n19.$CDATE.nc4 sondes.$CDATE.nc4" for file in $flist; do - ln -fs $GDASAPP_TESTDATA/lowres/$dpath/${oprefix}.$file $COM_OBS/ + ln -fs $GDASAPP_TESTDATA/lowres/$dpath/${oprefix}.$file $COM_OBS/${oprefix}.$file done # Link radiance bias correction files @@ -79,7 +79,7 @@ dpath=gdas.$gPDY/$gcyc/analysis/atmos mkdir -p $COM_ATMOS_ANALYSIS_PREV flist="amsua_n19.satbias.nc4 amsua_n19.satbias_cov.nc4 amsua_n19.tlapse.txt" for file in $flist; do - ln -fs $GDASAPP_TESTDATA/lowres/$dpath/$gprefix.$file $COM_ATMOS_ANALYSIS_PREV/ + ln -fs $GDASAPP_TESTDATA/lowres/$dpath/$gprefix.$file $COM_ATMOS_ANALYSIS_PREV/$gprefix.$file done # Link atmospheric background on gaussian grid @@ -87,7 +87,7 @@ dpath=gdas.$gPDY/$gcyc/model_data/atmos/history mkdir -p $COM_ATMOS_HISTORY_PREV flist="atmf006.nc" for file in $flist; do - ln -fs $GDASAPP_TESTDATA/lowres/$dpath/${gprefix}.${file} $COM_ATMOS_HISTORY_PREV/ + ln -fs $GDASAPP_TESTDATA/lowres/$dpath/${gprefix}.${file} $COM_ATMOS_HISTORY_PREV/${gprefix}.${file} done # Link atmospheric bacgkround on tiles @@ -96,7 +96,7 @@ COM_ATMOS_RESTART_PREV_DIRNAME=$(dirname $COM_ATMOS_RESTART_PREV) mkdir -p $COM_ATMOS_RESTART_PREV_DIRNAME flist="restart" for file in $flist; do - ln -fs $GDASAPP_TESTDATA/lowres/$dpath/$file $COM_ATMOS_RESTART_PREV_DIRNAME/ + ln -fs $GDASAPP_TESTDATA/lowres/$dpath/$file $COM_ATMOS_RESTART_PREV_DIRNAME/$file done @@ -125,10 +125,10 @@ done # Execute j-job -if [ $machine = 'HERA' ]; then +if [[ $machine = 'HERA' ]]; then + sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_INITIALIZE +elif [[ $machine = 'ORION' || $machine = 'HERCULES' ]]; then sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_INITIALIZE -elif [ $machine = 'ORION' ]; then - sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --partition=orion --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_INITIALIZE else ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_INITIALIZE fi diff --git a/test/atm/global-workflow/jjob_var_run.sh b/test/atm/global-workflow/jjob_var_run.sh index a78b1e10e..e13aa2daa 100755 --- a/test/atm/global-workflow/jjob_var_run.sh +++ b/test/atm/global-workflow/jjob_var_run.sh @@ -36,11 +36,11 @@ export PYTHONPATH machine=$(echo `grep 'machine=' $EXPDIR/config.base | cut -d"=" -f2` | tr -d '"') # Set NETCDF and UTILROOT variables (used in config.base) -if [ $machine = 'HERA' ]; then +if [[ $machine = 'HERA' ]]; then NETCDF=$( which ncdump ) export NETCDF export UTILROOT="/scratch2/NCEPDEV/ensemble/save/Walter.Kolczynski/hpc-stack/intel-18.0.5.274/prod_util/1.2.2" -elif [ $machine = 'ORION' ]; then +elif [[ $machine = 'ORION' || $machine = 'HERCULES' ]]; then ncdump=$( which ncdump ) NETCDF=$( echo "${ncdump}" | cut -d " " -f 3 ) export NETCDF @@ -48,10 +48,10 @@ elif [ $machine = 'ORION' ]; then fi # Execute j-job -if [ $machine = 'HERA' ]; then +if [[ $machine = 'HERA' ]]; then + sbatch --ntasks=6 --account=$ACCOUNT --qos=batch --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_RUN +elif [[ $machine = 'ORION' || $machine = 'HERCULES' ]]; then sbatch --ntasks=6 --account=$ACCOUNT --qos=batch --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_RUN -elif [ $machine = 'ORION' ]; then - sbatch --ntasks=6 --account=$ACCOUNT --qos=batch --partition=orion --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_RUN else ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_RUN fi diff --git a/test/soca/gw/CMakeLists.txt b/test/soca/gw/CMakeLists.txt index 2d7d059d8..5bcb9f71f 100644 --- a/test/soca/gw/CMakeLists.txt +++ b/test/soca/gw/CMakeLists.txt @@ -11,9 +11,14 @@ add_test(NAME test_gdasapp_soca_prep # Identify machine set(MACHINE "container") -IF (IS_DIRECTORY /work2/noaa/da) - set(MACHINE "orion") - set(PARTITION "orion") +IF (IS_DIRECTORY /work2) + IF (IS_DIRECTORY /apps/other) + set(MACHINE "hercules") + set(PARTITION "hercules") + ELSE() + set(MACHINE "orion") + set(PARTITION "orion") + ENDIF() ENDIF() IF (IS_DIRECTORY /scratch2/NCEPDEV/) set(MACHINE "hera") diff --git a/ush/detect_machine.sh b/ush/detect_machine.sh index 949526019..8610af15c 100755 --- a/ush/detect_machine.sh +++ b/ush/detect_machine.sh @@ -25,6 +25,8 @@ case $(hostname -f) in Orion-login-[1-4].HPC.MsState.Edu) MACHINE_ID=orion ;; ### orion1-4 + Hercules-login-[1-4].HPC.MsState.Edu) MACHINE_ID=hercules ;; ### hercules1-4 + cheyenne[1-6].cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne1-6 cheyenne[1-6].ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne1-6 chadmin[1-6].ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne1-6 diff --git a/ush/module-setup.sh b/ush/module-setup.sh index 469fd4a3c..ef7f73699 100755 --- a/ush/module-setup.sh +++ b/ush/module-setup.sh @@ -22,6 +22,13 @@ elif [[ $MACHINE_ID = orion* ]] ; then fi module purge +elif [[ $MACHINE_ID = hercules* ]] ; then + # We are on Hercules + if ( ! eval module help > /dev/null 2>&1 ) ; then + source /apps/lmod/init/bash + fi + module purge + elif [[ $MACHINE_ID = s4* ]] ; then # We are on SSEC Wisconsin S4 if ( ! eval module help > /dev/null 2>&1 ) ; then diff --git a/ush/soca/run_jjobs.py b/ush/soca/run_jjobs.py index 4e0212da2..f9b73c845 100755 --- a/ush/soca/run_jjobs.py +++ b/ush/soca/run_jjobs.py @@ -6,7 +6,7 @@ import argparse from datetime import datetime, timedelta -machines = {"container", "hera", "orion"} +machines = {"container", "hera", "orion", "hercules"} # Assume the default conda environement is gdassapp ENVS = {'JGDAS_GLOBAL_OCEAN_ANALYSIS_VRFY': 'eva'}