Skip to content

Commit

Permalink
Merge branch 'develop' into feature/landda5
Browse files Browse the repository at this point in the history
  • Loading branch information
jiaruidong2017 committed Aug 14, 2023
2 parents b46530b + 2102929 commit a4a84e7
Show file tree
Hide file tree
Showing 113 changed files with 10,988 additions and 2,206 deletions.
11 changes: 8 additions & 3 deletions .github/workflows/pynorms.yaml → .github/workflows/norms.yaml
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
name: Python Coding Norms
name: Coding Norms
on: [push]

jobs:
check_pynorms:
runs-on: ubuntu-latest
name: Check Python coding norms with pycodestyle
name: Check coding norms with pycodestyle and cpplint

steps:

Expand All @@ -20,4 +20,9 @@ jobs:
- name: Run pycodestyle
run: |
cd $GITHUB_WORKSPACE/GDASApp
pycodestyle -v --config ./.pycodestyle .
pycodestyle -v --config ./.pycodestyle ./ush ./scripts ./test
- name: Run C++ linter on utils
run: |
cd $GITHUB_WORKSPACE/GDASApp/utils/test/
./cpplint.py --quiet --recursive $GITHUB_WORKSPACE/GDASApp/utils
10 changes: 5 additions & 5 deletions .github/workflows/unittests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -24,16 +24,16 @@ jobs:
cd solo && pip install . && cd ..
cd r2d2 && pip install . && cd ..
- name: Checkout workflow
- name: Checkout wxflow
uses: actions/checkout@v3
with:
repository: NOAA-EMC/global-workflow
repository: NOAA-EMC/wxflow
ref: develop
path: global-workflow
path: wxflow

- name: Install workflow utils
- name: Install wxflow
run: |
cd global-workflow/ush/python/pygw
cd wxflow
pip install .
- name: Checkout
Expand Down
17 changes: 12 additions & 5 deletions .github/workflows/unittests_g-w.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -25,18 +25,25 @@ jobs:
cd r2d2 && pip install . && cd ..
sudo mkdir -p /work/noaa # to trick workflow into thinking this is RDHPCS Orion
- name: Checkout workflow
- name: Checkout wxflow
uses: actions/checkout@v3
with:
repository: NOAA-EMC/global-workflow
repository: NOAA-EMC/wxflow
ref: develop
path: global-workflow
path: wxflow

- name: Install workflow utils
- name: Install wxflow
run: |
cd global-workflow/ush/python/pygw
cd wxflow
pip install .
- name: Checkout workflow
uses: actions/checkout@v3
with:
repository: NOAA-EMC/global-workflow
ref: develop
path: global-workflow

- name: Checkout GDASApp
uses: actions/checkout@v3
with:
Expand Down
11 changes: 7 additions & 4 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ if(BUILD_GDASBUNDLE)

# External (required) observation operators
option("BUNDLE_SKIP_CRTM" "Don't build CRTM" "OFF") # Don't build crtm unless user passes -DBUNDLE_SKIP_CRTM=OFF
ecbuild_bundle( PROJECT crtm GIT "https://github.com/ADCollard/crtm.git" TAG v2.3-jedi.3_fix )
ecbuild_bundle( PROJECT crtm GIT "https://github.com/JCSDA/crtm.git" TAG v2.4.1-jedi.1 )

# Build GSI-B
option(BUILD_GSIBEC "Build GSI-B" OFF)
Expand All @@ -75,7 +75,7 @@ if(BUILD_GDASBUNDLE)
ecbuild_bundle( PROJECT vader GIT "https://github.com/jcsda/vader.git" BRANCH develop )
ecbuild_bundle( PROJECT saber GIT "https://github.com/jcsda/saber.git" BRANCH develop )
ecbuild_bundle( PROJECT ioda GIT "https://github.com/jcsda/ioda.git" BRANCH develop )
ecbuild_bundle( PROJECT ufo GIT "https://github.com/jcsda/ufo.git" BRANCH feature/gdasapp_ufo )
ecbuild_bundle( PROJECT ufo GIT "https://github.com/jcsda/ufo.git" BRANCH develop )

# FMS and FV3 dynamical core
ecbuild_bundle( PROJECT fms GIT "https://github.com/jcsda/FMS.git" BRANCH release-stable )
Expand All @@ -94,7 +94,10 @@ if(BUILD_GDASBUNDLE)
endif()
ecbuild_bundle( PROJECT gsw GIT "https://github.com/jcsda-internal/GSW-Fortran.git" BRANCH develop )
ecbuild_bundle( PROJECT mom6 GIT "https://github.com/jcsda-internal/MOM6.git" BRANCH main-ecbuild RECURSIVE )
ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH develop )
ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH feature/change_mask_value )

# Build JEDI/DA or other peripherals
ecbuild_bundle( PROJECT gdas-utils SOURCE "./utils" )

# Build IODA converters
option(BUILD_IODA_CONVERTERS "Build IODA Converters" ON)
Expand All @@ -104,7 +107,7 @@ if(BUILD_GDASBUNDLE)

# Land associated repositories
ecbuild_bundle( PROJECT land-imsproc GIT "https://github.com/NOAA-PSL/land-IMS_proc.git" TAG 6373819 )
ecbuild_bundle( PROJECT land-jediincr GIT "https://github.com/NOAA-PSL/land-apply_jedi_incr.git" TAG ced6576 )
ecbuild_bundle( PROJECT land-jediincr GIT "https://github.com/NOAA-PSL/land-apply_jedi_incr.git" TAG 2923344)

# ioda, ufo, fv3-jedi, and saber test data
#---------------------------------
Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,4 +7,4 @@ The one app to rule them all
[![Hera](https://github.com/NOAA-EMC/GDASApp/actions/workflows/hera.yaml/badge.svg)](https://github.com/NOAA-EMC/GDASApp/actions/workflows/hera.yaml)

[![Unit Tests on GitHub CI](https://github.com/NOAA-EMC/GDASApp/actions/workflows/unittests.yaml/badge.svg)](https://github.com/NOAA-EMC/GDASApp/actions/workflows/unittests.yaml)
[![Python Coding Norms](https://github.com/NOAA-EMC/GDASApp/actions/workflows/pynorms.yaml/badge.svg)](https://github.com/NOAA-EMC/GDASApp/actions/workflows/pynorms.yaml)
[![Coding Norms](https://github.com/NOAA-EMC/GDASApp/actions/workflows/norms.yaml/badge.svg)](https://github.com/NOAA-EMC/GDASApp/actions/workflows/norms.yaml)
7 changes: 4 additions & 3 deletions build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ usage() {
# Defaults:
INSTALL_PREFIX=""
CMAKE_OPTS=""
BUILD_TARGET="${MACHINE_ID}"
BUILD_TARGET="${MACHINE_ID:-'localhost'}"
BUILD_VERBOSE="NO"
CLONE_JCSDADATA="NO"
CLEAN_BUILD="NO"
Expand Down Expand Up @@ -76,7 +76,6 @@ case ${BUILD_TARGET} in
module use $dir_root/modulefiles
module load GDAS/$BUILD_TARGET
CMAKE_OPTS+=" -DMPIEXEC_EXECUTABLE=$MPIEXEC_EXEC -DMPIEXEC_NUMPROC_FLAG=$MPIEXEC_NPROC -DBUILD_GSIBEC=ON"
CMAKE_OPTS+=" -DCLONE_JCSDADATA=$CLONE_JCSDADATA"
module list
;;
$(hostname))
Expand All @@ -87,6 +86,8 @@ case ${BUILD_TARGET} in
;;
esac

CMAKE_OPTS+=" -DCLONE_JCSDADATA=$CLONE_JCSDADATA"

BUILD_DIR=${BUILD_DIR:-$dir_root/build}
if [[ $CLEAN_BUILD == 'YES' ]]; then
[[ -d ${BUILD_DIR} ]] && rm -rf ${BUILD_DIR}
Expand Down Expand Up @@ -114,7 +115,7 @@ set -x
if [[ $BUILD_JCSDA == 'YES' ]]; then
make -j ${BUILD_JOBS:-6} VERBOSE=$BUILD_VERBOSE
else
builddirs="fv3-jedi soca iodaconv land-imsproc land-jediincr"
builddirs="fv3-jedi soca iodaconv land-imsproc land-jediincr gdas-utils"
for b in $builddirs; do
cd $b
make -j ${BUILD_JOBS:-6} VERBOSE=$BUILD_VERBOSE
Expand Down
86 changes: 75 additions & 11 deletions ci/stable_driver.sh
Original file line number Diff line number Diff line change
Expand Up @@ -47,15 +47,25 @@ case ${TARGET} in
esac

# ==============================================================================
# clone a fresh copy of the develop branch
datestr="$(date +%Y%m%d)"
repo_url="https://github.com/NOAA-EMC/GDASApp.git"
workflow_url="https://github.com/NOAA-EMC/global-workflow.git"
stableroot=$GDAS_CI_ROOT/stable

mkdir -p $stableroot/$datestr
cd $stableroot/$datestr
git clone $repo_url
cd GDASApp

# clone global workflow develop branch
git clone $workflow_url

# run checkout script for all other components
cd $stableroot/$datestr/global-workflow/sorc
./checkout.sh -u

# checkout develop
cd gdas.cd
git checkout develop
git pull

# ==============================================================================
# run ecbuild to get the repos cloned
Expand All @@ -67,21 +77,75 @@ rm -rf build

# ==============================================================================
# update the hashes to the most recent
$my_dir/stable_mark.sh $stableroot/$datestr/GDASApp
gdasdir=$stableroot/$datestr/global-workflow/sorc/gdas.cd
$my_dir/stable_mark.sh $gdasdir

# ==============================================================================
# run the automated testing
$my_dir/run_ci.sh -d $stableroot/$datestr/GDASApp -o $stableroot/$datestr/output
$my_dir/run_gw_ci.sh -d $stableroot/$datestr/global-workflow -o $stableroot/$datestr/output
ci_status=$?
total=0
if [ $ci_status -eq 0 ]; then
# push a new commit to the stable branch
cd $stableroot/$datestr/GDASApp
git push origin --delete feature/stable-nightly
git checkout -b feature/stable-nightly
cd $gdasdir
# copy the CMakeLists file for safe keeping
cp $gdasdir/CMakeLists.txt $gdasdir/CMakeLists.txt.new
total=$(($total+$?))
if [ $total -ne 0 ]; then
echo "Unable to cp CMakeLists" >> $stableroot/$datestr/output
fi
# checkout feature/stable-nightly
git stash
total=$(($total+$?))
if [ $total -ne 0 ]; then
echo "Unable to cp CMakeLists" >> $stableroot/$datestr/output
fi
git checkout feature/stable-nightly
total=$(($total+$?))
if [ $total -ne 0 ]; then
echo "Unable to cp CMakeLists" >> $stableroot/$datestr/output
fi
# merge in develop
git merge develop
total=$(($total+$?))
if [ $total -ne 0 ]; then
echo "Unable to merge develop" >> $stableroot/$datestr/output
fi
# force move the copy to the original path of CMakeLists.txt
/bin/mv -f $gdasdir/CMakeLists.txt.new $gdasdir/CMakeLists.txt
total=$(($total+$?))
if [ $total -ne 0 ]; then
echo "Unable to mv CMakeLists" >> $stableroot/$datestr/output
fi
# commit this change and push
git add CMakeLists.txt
git commit -m "Update to new stable build on $datestr"
total=$(($total+$?))
if [ $total -ne 0 ]; then
echo "Unable to add CMakeLists to commit" >> $stableroot/$datestr/output
fi
git diff-index --quiet HEAD || git commit -m "Update to new stable build on $datestr"
total=$(($total+$?))
caution=""
if [ $total -ne 0 ]; then
echo "Unable to commit" >> $stableroot/$datestr/output
fi
git push --set-upstream origin feature/stable-nightly
echo "Stable branch updated"
total=$(($total+$?))
if [ $total -ne 0 ]; then
echo "Unable to push" >> $stableroot/$datestr/output
fi
if [ $total -ne 0 ]; then
echo "Issue merging with develop. please manually fix"
PEOPLE="[email protected] [email protected] [email protected]"
SUBJECT="Problem updating feature/stable-nightly branch of GDASApp"
BODY=$stableroot/$datestr/output_stable_nightly
cat > $BODY << EOF
Problem updating feature/stable-nightly branch of GDASApp. Please check $stableroot/$datestr/GDASApp
EOF
mail -r "Darth Vader - NOAA Affiliate <[email protected]>" -s "$SUBJECT" "$PEOPLE" < $BODY
else
echo "Stable branch updated"
fi
else
# do nothing
echo "Testing failed, stable branch will not be updated"
Expand Down
28 changes: 14 additions & 14 deletions modulefiles/GDAS/hera.lua
Original file line number Diff line number Diff line change
Expand Up @@ -8,44 +8,44 @@ local pkgNameVer = myModuleFullName()

conflict(pkgName)

prepend_path("MODULEPATH", '/scratch1/NCEPDEV/global/spack-stack/spack-stack-v1/envs/skylab-3.0.0-intel-2021.5.0/install/modulefiles/Core')
prepend_path("MODULEPATH", '/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.3.1/envs/unified-env/install/modulefiles/Core')
prepend_path("MODULEPATH", '/scratch1/NCEPDEV/da/python/opt/modulefiles/stack')

-- below two lines get us access to the spack-stack modules
load("stack-intel/2021.5.0")
load("stack-intel-oneapi-mpi/2021.5.1")
-- JCSDA has 'jedi-fv3-env/1.0.0', but we should load these manually as needed
-- JCSDA has 'jedi-fv3-env/unified-dev', but we should load these manually as needed
load("cmake/3.23.1")
load("curl/7.29.0")
load("gettext/0.19.8.1")
load("libunistring/0.9.10")
load("libidn2/2.3.0")
load("pcre2/10.39")
load("pcre2/10.42")
load("zlib/1.2.13")
load("git/2.38.1")
load("pkg-config/0.27.1")
load("hdf5/1.12.1")
load("hdf5/1.12.2")
load("parallel-netcdf/1.12.2")
load("netcdf-c/4.8.1")
load("netcdf-c/4.9.2")
load("nccmp/1.9.0.1")
load("netcdf-fortran/4.5.4")
load("netcdf-fortran/4.6.0")
load("nco/5.0.6")
load("parallelio/2.5.7")
load("parallelio/2.5.9")
load("wget/1.14")
load("boost/1.78.0")
load("bufr/11.7.1")
load("git-lfs/2.10.0")
load("ecbuild/3.6.5")
load("ecbuild/3.7.2")
load("openjpeg/2.3.1")
load("eccodes/2.27.0")
load("eigen/3.4.0")
load("openblas/0.3.19")
load("eckit/1.20.2")
load("eckit/1.23.0")
load("fftw/3.3.10")
load("fckit/0.9.5")
load("fiat/1.0.0")
load("ectrans/1.1.0")
load("atlas/0.31.1")
load("fckit/0.10.1")
load("fiat/1.1.0")
load("ectrans/1.2.0")
load("atlas/0.33.0")
load("sp/2.3.3")
load("gsl-lite/0.37.0")
load("libjpeg/2.1.0")
Expand Down Expand Up @@ -80,7 +80,7 @@ setenv('MPIEXEC_EXEC', mpiexec)
setenv('MPIEXEC_NPROC', mpinproc)

setenv('R2D2_CONFIG', '/scratch1/NCEPDEV/stmp4/Cory.R.Martin/R2D2_SHARED/config_hera.yaml')
setenv("CRTM_FIX","/scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/fix/crtm")
setenv("CRTM_FIX","/scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/fix/crtm/2.4.0")
setenv("GDASAPP_TESTDATA","/scratch1/NCEPDEV/da/Cory.R.Martin/CI/GDASApp/data")
prepend_path("PATH","/scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/intel-18.0.5.274/prod_util/1.2.2/bin")

Expand Down
26 changes: 13 additions & 13 deletions modulefiles/GDAS/orion.lua
Original file line number Diff line number Diff line change
Expand Up @@ -6,41 +6,41 @@ local pkgName = myModuleName()
local pkgVersion = myModuleVersion()
local pkgNameVer = myModuleFullName()

prepend_path("MODULEPATH", '/work/noaa/da/role-da/spack-stack/spack-stack-v1/envs/skylab-3.0.0-intel-2022.0.2/install/modulefiles/Core')
prepend_path("MODULEPATH", '/work/noaa/epic-ps/role-epic-ps/spack-stack/spack-stack-1.3.1/envs/unified-env/install/modulefiles/Core')

prepend_path("MODULEPATH", '/work2/noaa/da/python/opt/modulefiles/stack')

-- below two lines get us access to the spack-stack modules
load("stack-intel/2022.0.2")
load("stack-intel-oneapi-mpi/2021.5.1")
-- JCSDA has 'jedi-fv3-env/1.0.0', but we should load these manually as needed
-- JCSDA has 'jedi-fv3-env/unified-dev', but we should load these manually as needed
load("cmake/3.22.1")
load("zlib/1.2.13")
load("curl/7.85.0")
load("git/2.28.0")
load("pkg-config/0.27.1")
load("hdf5/1.12.1")
load("hdf5/1.12.2")
load("parallel-netcdf/1.12.2")
load("netcdf-c/4.8.1")
load("netcdf-c/4.9.2")
load("nccmp/1.9.0.1")
load("netcdf-fortran/4.5.4")
load("netcdf-fortran/4.6.0")
load("nco/5.0.6")
load("parallelio/2.5.7")
load("parallelio/2.5.9")
load("wget/1.14")
load("boost/1.72.0")
load("bufr/11.7.1")
load("git-lfs/2.12.0")
load("ecbuild/3.6.5")
load("ecbuild/3.7.2")
load("openjpeg/2.3.1")
load("eccodes/2.27.0")
load("eigen/3.4.0")
load("openblas/0.3.19")
load("eckit/1.20.2")
load("eckit/1.23.0")
load("fftw/3.3.10")
load("fckit/0.9.5")
load("fiat/1.0.0")
load("ectrans/1.1.0")
load("atlas/0.31.1")
load("fckit/0.10.1")
load("fiat/1.1.0")
load("ectrans/1.2.0")
load("atlas/0.33.0")
load("sp/2.3.3")
load("gsl-lite/0.37.0")
load("libjpeg/2.1.0")
Expand Down Expand Up @@ -78,7 +78,7 @@ setenv('MPIEXEC_EXEC', mpiexec)
setenv('MPIEXEC_NPROC', mpinproc)

setenv('R2D2_CONFIG', '/work2/noaa/da/cmartin/GDASApp/R2D2_SHARED/config_orion.yaml')
setenv("CRTM_FIX","/work2/noaa/da/cmartin/GDASApp/fix/crtm")
setenv("CRTM_FIX","/work2/noaa/da/cmartin/GDASApp/fix/crtm/2.4.0")
setenv("GDASAPP_TESTDATA","/work2/noaa/da/cmartin/CI/GDASApp/data")
prepend_path("PATH","/apps/contrib/NCEP/libs/hpc-stack/intel-2018.4/prod_util/1.2.2/bin")

Expand Down
Loading

0 comments on commit a4a84e7

Please sign in to comment.