Merge branch 'develop' into feature/jack-bauer

.github/workflows/unittests.yaml

@@ -24,16 +24,16 @@ jobs:
         cd solo && pip install . && cd ..
         cd r2d2 && pip install . && cd ..
 
-    - name: Checkout workflow
+    - name: Checkout wxflow
       uses: actions/checkout@v3
       with:
-        repository: NOAA-EMC/global-workflow
+        repository: NOAA-EMC/wxflow
         ref: develop
-        path: global-workflow
+        path: wxflow
 
-    - name: Install workflow utils
+    - name: Install wxflow
       run: |
-        cd global-workflow/ush/python/pygw
+        cd wxflow
         pip install .
 
     - name: Checkout

.github/workflows/unittests_g-w.yaml

@@ -25,18 +25,25 @@ jobs:
         cd r2d2 && pip install . && cd ..
         sudo mkdir -p /work/noaa # to trick workflow into thinking this is RDHPCS Orion
 
-    - name: Checkout workflow
+    - name: Checkout wxflow
       uses: actions/checkout@v3
       with:
-        repository: NOAA-EMC/global-workflow
+        repository: NOAA-EMC/wxflow
         ref: develop
-        path: global-workflow
+        path: wxflow
 
-    - name: Install workflow utils
+    - name: Install wxflow
       run: |
-        cd global-workflow/ush/python/pygw
+        cd wxflow
         pip install .
 
+    - name: Checkout workflow
+      uses: actions/checkout@v3
+      with:
+        repository: NOAA-EMC/global-workflow
+        ref: develop
+        path: global-workflow
+
     - name: Checkout GDASApp
       uses: actions/checkout@v3
       with:

scripts/exgdas_global_marine_analysis_post.py

@@ -23,7 +23,7 @@
 import shutil
 import logging
 from datetime import datetime, timedelta
-from pygw.file_utils import FileHandler
+from wxflow import FileHandler
 
 
 # TODO: Move this somewhere else?

scripts/exgdas_global_marine_analysis_prep.py

@@ -32,10 +32,7 @@
 from netCDF4 import Dataset
 import xarray as xr
 import numpy as np
-from pygw.attrdict import AttrDict
-from pygw.template import Template, TemplateConstants
-from pygw.yaml_file import YAMLFile
-from pygw.file_utils import FileHandler
+from wxflow import (AttrDict, Template, TemplateConstants, YAMLFile, FileHandler)
 
 # set up logger
 logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', level=logging.INFO, datefmt='%Y-%m-%d %H:%M:%S')

test/aero/genyaml_3dvar.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 # generate YAML from a template
-# using pygw YAML tools
+# using wxflow YAML tools
 bindir=$1
 srcdir=$2
 
@@ -27,9 +27,7 @@ mkdir -p $DATA
 
 # run some python code to generate the YAML
 python3 - <<EOF
-from pygw.attrdict import AttrDict
-from pygw.template import Template, TemplateConstants
-from pygw.yaml_file import YAMLFile
+from wxflow import AttrDict, Template, TemplateConstants, YAMLFile
 import datetime
 
 config = YAMLFile(path='$YAMLin')

test/atm/global-workflow/jjob_ens_final.sh

@@ -25,8 +25,8 @@ export NMEM_ENS=3
 export ACCOUNT=da-cpu
 
 # Set python path for workflow utilities and tasks
-pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}"
+wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
+PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
 export PYTHONPATH
 
 # Detemine machine from config.base

test/atm/global-workflow/jjob_ens_init.sh

@@ -30,8 +30,8 @@ source "${HOMEgfs}/ush/preamble.sh"
 source "${HOMEgfs}/parm/config/gfs/config.com"
 
 # Set python path for workflow utilities and tasks
-pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}"
+wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
+PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
 export PYTHONPATH
 
 # Detemine machine from config.base

test/atm/global-workflow/jjob_ens_run.sh

@@ -28,8 +28,8 @@ export NMEM_ENS=3
 export ACCOUNT=da-cpu
 
 # Set python path for workflow utilities and tasks
-pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}"
+wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
+PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
 export PYTHONPATH
 
 # Detemine machine from config.base

test/atm/global-workflow/jjob_var_final.sh

@@ -25,8 +25,8 @@ export NMEM_ENS=0
 export ACCOUNT=da-cpu
 
 # Set python path for workflow utilities and tasks
-pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}"
+wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
+PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
 export PYTHONPATH
 
 # Detemine machine from config.base

test/atm/global-workflow/jjob_var_init.sh

@@ -30,8 +30,8 @@ source "${HOMEgfs}/ush/preamble.sh"
 source "${HOMEgfs}/parm/config/gfs/config.com"
 
 # Set python path for workflow utilities and tasks
-pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}"
+wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
+PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
 export PYTHONPATH
 
 # Detemine machine from config.base

test/atm/global-workflow/jjob_var_run.sh

@@ -28,8 +28,8 @@ export NMEM_ENS=0
 export ACCOUNT=da-cpu
 
 # Set python path for workflow utilities and tasks
-pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}"
+wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
+PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
 export PYTHONPATH
 
 # Detemine machine from config.base

test/atm/global-workflow/run_jedi_exe_3denvar.sh

@@ -33,8 +33,8 @@ fi
 # Setup python path for workflow utilities and tasks
 export HOMEgfs=$srcdir/../../ # TODO: HOMEgfs had to be hard-coded in config
 echo $HOMEgfs
-pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}"
+wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
+PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
 export PYTHONPATH
 
 # Create test run directory

test/atm/global-workflow/run_jedi_exe_3dhofx.sh

@@ -33,8 +33,8 @@ fi
 # Setup python path for workflow utilities and tasks
 export HOMEgfs=$srcdir/../../ # TODO: HOMEgfs had to be hard-coded in config
 echo $HOMEgfs
-pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}"
+wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
+PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
 export PYTHONPATH
 
 # Create test run directory

test/atm/global-workflow/run_jedi_exe_3dvar.sh

@@ -33,8 +33,8 @@ fi
 # Setup python path for workflow utilities and tasks
 export HOMEgfs=$srcdir/../../ # TODO: HOMEgfs had to be hard-coded in config
 echo $HOMEgfs
-pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}"
+wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
+PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
 export PYTHONPATH
 
 # Create test run directory

test/atm/global-workflow/run_jedi_exe_letkf.sh

@@ -33,8 +33,8 @@ fi
 # Setup python path for workflow utilities and tasks
 export HOMEgfs=$srcdir/../../ # TODO: HOMEgfs had to be hard-coded in config
 echo $HOMEgfs
-pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}"
+wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
+PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
 export PYTHONPATH
 
 # Create test run directory

test/check_valid_yaml.py

@@ -4,7 +4,7 @@
 import os
 import pathlib
 import sys
-from pygw.yaml_file import YAMLFile
+from wxflow import YAMLFile
 
 
 def check_valid_yaml(repodir):

test/soca/CMakeLists.txt

@@ -69,7 +69,7 @@ add_test(NAME test_gdasapp_soca_obsdb
          WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/obs/)
 set_tests_properties(test_gdasapp_soca_obsdb
     PROPERTIES
-    ENVIRONMENT "PYTHONPATH=${PROJECT_BINARY_DIR}/ush:${PROJECT_SOURCE_DIR}/../../ush/python/pygw/src:$ENV{PYTHONPATH};SOCA_TEST_OBS=${PROJECT_SOURCE_DIR}/test/soca/testdata")
+    ENVIRONMENT "PYTHONPATH=${PROJECT_BINARY_DIR}/ush:${PROJECT_SOURCE_DIR}/../../ush/python/wxflow/src:$ENV{PYTHONPATH};SOCA_TEST_OBS=${PROJECT_SOURCE_DIR}/test/soca/testdata")
 
 # Test exgdas scripts from the global-worflow
 if (WORKFLOW_TESTS)

test/soca/gw/CMakeLists.txt

@@ -8,7 +8,7 @@ add_test(NAME test_gdasapp_soca_prep
          WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw)
        set_tests_properties(test_gdasapp_soca_prep
       PROPERTIES
-      ENVIRONMENT "PYTHONPATH=${PROJECT_BINARY_DIR}/ush:${PROJECT_SOURCE_DIR}/../../ush/python/pygw/src:$ENV{PYTHONPATH}")
+      ENVIRONMENT "PYTHONPATH=${PROJECT_BINARY_DIR}/ush:${PROJECT_SOURCE_DIR}/../../ush/python/wxflow/src:$ENV{PYTHONPATH}")
 
 # Identify machine
 set(MACHINE "container")

ush/ioda/gen_bufr2ioda_yaml.py

@@ -5,8 +5,7 @@
 # and certain configuration parameters
 import argparse
 import os
-from pygw.template import Template, TemplateConstants
-from pygw.yaml_file import YAMLFile
+from wxflow import Template, TemplateConstants, YAMLFile
 
 
 # list of satellite radiance BUFR files that need split by SatId

ush/run_jedi_exe.py

@@ -7,8 +7,8 @@
 import subprocess
 import sys
 import yaml
-import pygw
-from pygw.yaml_file import parse_j2yaml, save_as_yaml
+import wxflow
+from wxflow import parse_j2yaml, save_as_yaml
 
 
 def export_envar(yamlfile, bashout):

ush/soca/run_jjobs.py

@@ -81,7 +81,7 @@ def export_env_vars_script(self):
         self.f.write(f"export CDATE='{CDATE}'\n")
 
         # Add to python environement
-        self.f.write("PYTHONPATH=${HOMEgfs}/ush/python/pygw/src:${PYTHONPATH}\n")
+        self.f.write("PYTHONPATH=${HOMEgfs}/ush/python/wxflow/src:${PYTHONPATH}\n")
 
     def setupexpt(self):
         """

ush/socaincr2mom6.py

@@ -6,7 +6,7 @@
 from netCDF4 import Dataset
 from scipy.interpolate import griddata
 import ufsda
-from pygw.yaml_file import YAMLFile
+from wxflow import YAMLFile
 
 
 def socaincr2mom6(incr, bkg, grid, incr_out, nsst_yaml=None):

ush/ufsda/archive.py

@@ -5,7 +5,7 @@
 import shutil
 import datetime as dt
 import ufsda
-from pygw.yaml_file import YAMLFile
+from wxflow import YAMLFile
 
 __all__ = ['atm_diags']
 

ush/ufsda/genYAML.py

@@ -8,8 +8,7 @@
 import os
 import re
 import yaml
-from pygw.template import Template, TemplateConstants
-from pygw.yaml_file import YAMLFile
+from wxflow import Template, TemplateConstants, YAMLFile
 
 
 def genYAML(yamlconfig, output=None):

ush/ufsda/stage.py

@@ -12,7 +12,7 @@
 import logging
 import glob
 import numpy as np
-from pygw.yaml_file import YAMLFile, parse_yaml, parse_j2yaml
+from wxflow import YAMLFile, parse_yaml, parse_j2yaml
 import ufsda.soca_utils
 
 __all__ = ['atm_background', 'atm_obs', 'bias_obs', 'background', 'background_ens', 'fv3jedi', 'obs', 'berror', 'gdas_fix', 'gdas_single_cycle']

ush/ufsda/yamltools.py

@@ -5,7 +5,7 @@
 from solo.yaml_file import YAMLFile
 from solo.template import TemplateConstants, Template
 from ufsda.misc_utils import isTrue
-import pygw
+import wxflow
 
 logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s',
                     level=logging.INFO, datefmt='%Y-%m-%d %H:%M:%S')
@@ -36,7 +36,7 @@ def save_check(config, target, app='var'):
     config['cost function']['observations']['observers'] = cleaned_obs_spaces
 
     # save cleaned yaml
-    pygw.yaml_file.save_as_yaml(config, target)
+    wxflow.save_as_yaml(config, target)
 
 
 def parse_config(input_config_dict, template=None, clean=True):