Skip to content

Commit

Permalink
Merge branch 'develop' into feature/marine-monthly-bufr2ioda
Browse files Browse the repository at this point in the history
  • Loading branch information
@guillaumevernieres
guillaumevernieres committed Jan 18, 2024
2 parents 5ab7ac4 + f63fe7e commit de86fb4
Show file tree
Hide file tree
Showing 74 changed files with 738 additions and 397 deletions.
65 changes: 36 additions & 29 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -146,33 +146,40 @@ build*/
install*/

# Ignore the following bundle repositories
ecbuild/
atlas/
crtm/
eckit/
fckit/
femps/
fms/
fv3/
fv3-jedi/
fv3-jedi-data/
fv3-jedi-lm/
gsibec/
gsw/
icepack/
/ioda/
ioda-data/
iodaconv/
jedicmake/
land-imsproc/
land-jediincr/
mom6/
oops/
saber/
saber-data/
/soca/
test-data-release/
ufo/
ufo-data/
vader/
/ecbuild
/atlas
/crtm
/eckit
/fckit
/femps
/fms
/fv3
/fv3-jedi
/fv3-jedi-data
/fv3-jedi-lm
/gsibec
/gsw
/icepack
/ioda
/ioda-data
/iodaconv
/jedicmake
/land-imsproc
/land-jediincr
/mom6
/oops
/saber
/saber-data
/soca
/test-data-release
/ufo
/ufo-data
/vader
/gdas-utils
/sorc/fv3-jedi-data
/sorc/ufo-data
/sorc/ioda-data
/sorc/test-data-release
/sorc/gdas
/sorc/gdas-utils

78 changes: 78 additions & 0 deletions .gitmodules
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
[submodule "sorc/jedicmake"]
path = sorc/jedicmake
url = https://github.com/jcsda/jedi-cmake.git
[submodule "sorc/gsibec"]
path = sorc/gsibec
url = https://github.com/GEOS-ESM/GSIbec.git
branch = develop
[submodule "sorc/gsw"]
path = sorc/gsw
url = https://github.com/jcsda-internal/GSW-Fortran.git
branch = develop
[submodule "sorc/oops"]
path = sorc/oops
url = https://github.com/jcsda/oops.git
branch = develop
[submodule "sorc/vader"]
path = sorc/vader
url = https://github.com/jcsda/vader.git
branch = develop
[submodule "sorc/saber"]
path = sorc/saber
url = https://github.com/jcsda/saber.git
branch = develop
[submodule "sorc/ioda"]
path = sorc/ioda
url = https://github.com/jcsda/ioda.git
branch = develop
[submodule "sorc/ufo"]
path = sorc/ufo
url = https://github.com/jcsda/ufo.git
branch = develop
[submodule "sorc/fv3"]
path = sorc/fv3
url = https://github.com/jcsda/GFDL_atmos_cubed_sphere.git
branch = release-stable
[submodule "sorc/femps"]
path = sorc/femps
url = https://github.com/jcsda/femps.git
branch = develop
[submodule "sorc/fv3-jedi-lm"]
path = sorc/fv3-jedi-lm
url = https://github.com/jcsda/fv3-jedi-linearmodel.git
branch = develop
[submodule "sorc/fv3-jedi"]
path = sorc/fv3-jedi
url = https://github.com/jcsda/fv3-jedi.git
branch = develop
[submodule "sorc/icepack"]
path = sorc/icepack
url = https://github.com/JCSDA-internal/Icepack.git
branch = feature/ecbuild-new
[submodule "sorc/mom6"]
path = sorc/mom6
url = https://github.com/jcsda-internal/MOM6.git
branch = main-ecbuild
[submodule "sorc/soca"]
path = sorc/soca
url = https://github.com/jcsda-internal/soca.git
branch = develop
[submodule "sorc/iodaconv"]
path = sorc/iodaconv
url = https://github.com/JCSDA-internal/ioda-converters.git
branch = develop
[submodule "sorc/land-imsproc"]
path = sorc/land-imsproc
url = https://github.com/NOAA-PSL/land-IMS_proc.git
branch = develop
[submodule "sorc/land-jediincr"]
path = sorc/land-jediincr
url = https://github.com/NOAA-PSL/land-apply_jedi_incr.git
branch = develop
[submodule "sorc/crtm"]
path = sorc/crtm
url = https://github.com/jcsda/crtm.git
[submodule "sorc/fms"]
path = sorc/fms
url = https://github.com/jcsda/fms.git
branch = release-stable
129 changes: 1 addition & 128 deletions CMakeLists.txt
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# ------------------------------------------------------------------------- #
# Application for all the components needed for the GDAS system #
# Application for the GDAS system #
# ------------------------------------------------------------------------- #

# Check for minimim cmake requirement
Expand Down Expand Up @@ -36,135 +36,8 @@ option(BUILD_GDASBUNDLE "Build GDAS Bundle" ON)
option(CLONE_JCSDADATA "Clone JCSDA test data repositories" OFF)
option(WORKFLOW_TESTS "Include global-workflow dependent tests" OFF)

# Initialize bundle
# -----------------
ecbuild_bundle_initialize()

# Build bundle source code.
if(BUILD_GDASBUNDLE)

# jedi-cmake
ecbuild_bundle( PROJECT jedicmake GIT "https://github.com/jcsda/jedi-cmake.git" TAG 1.4.0 RECURSIVE )
include( jedicmake/cmake/Functions/git_functions.cmake )

# ECMWF libraries
option("BUNDLE_SKIP_ECKIT" "Don't build eckit" "ON" ) # Skip eckit build unless user passes -DBUNDLE_SKIP_ECKIT=OFF
option("BUNDLE_SKIP_FCKIT" "Don't build fckit" "ON") # Skip fckit build unless user passes -DBUNDLE_SKIP_FCKIT=OFF
option("BUNDLE_SKIP_ATLAS" "Don't build atlas" "ON") # Skip atlas build unless user passes -DBUNDLE_SKIP_ATLAS=OFF

# turn off optional OOPS toy models
option( ENABLE_LORENZ95_MODEL "Build LORENZ95 toy model" OFF )
option( ENABLE_QG_MODEL "Build QG toy model" OFF )

ecbuild_bundle( PROJECT eckit GIT "https://github.com/ecmwf/eckit.git" TAG 1.16.0 )
ecbuild_bundle( PROJECT fckit GIT "https://github.com/ecmwf/fckit.git" TAG 0.9.2 )
ecbuild_bundle( PROJECT atlas GIT "https://github.com/ecmwf/atlas.git" TAG 0.35.0 )

# External (required) observation operators
option("BUNDLE_SKIP_CRTM" "Don't build CRTM" "OFF") # Don't build crtm unless user passes -DBUNDLE_SKIP_CRTM=OFF
ecbuild_bundle( PROJECT crtm GIT "https://github.com/JCSDA/crtm.git" TAG v2.4.1-jedi.1 )

# Build GSI-B
option(BUILD_GSIBEC "Build GSI-B" OFF)
if(BUILD_GSIBEC)
ecbuild_bundle( PROJECT gsibec GIT "https://github.com/GEOS-ESM/GSIbec.git" TAG 1.1.2 )
endif()

# Gibbs seawater
ecbuild_bundle( PROJECT gsw GIT "https://github.com/jcsda-internal/GSW-Fortran.git" BRANCH develop )

# Core JEDI repositories
ecbuild_bundle( PROJECT oops GIT "https://github.com/jcsda/oops.git" BRANCH develop)
ecbuild_bundle( PROJECT vader GIT "https://github.com/jcsda/vader.git" BRANCH develop )
ecbuild_bundle( PROJECT saber GIT "https://github.com/jcsda/saber.git" BRANCH develop )
option(ENABLE_IODA_DATA "Obtain ioda test data from ioda-data repository (vs tarball)" ON)
ecbuild_bundle( PROJECT ioda GIT "https://github.com/jcsda/ioda.git" BRANCH develop )
option(ENABLE_UFO_DATA "Obtain ufo test data from ufo-data repository (vs tarball)" ON)
ecbuild_bundle( PROJECT ufo GIT "https://github.com/jcsda/ufo.git" BRANCH develop )

# FMS and FV3 dynamical core
ecbuild_bundle( PROJECT fms GIT "https://github.com/jcsda/FMS.git" BRANCH release-stable )
ecbuild_bundle( PROJECT fv3 GIT "https://github.com/jcsda/GFDL_atmos_cubed_sphere.git" BRANCH release-stable )

# fv3-jedi and associated repositories
ecbuild_bundle( PROJECT femps GIT "https://github.com/jcsda/femps.git" BRANCH develop )
ecbuild_bundle( PROJECT fv3-jedi-lm GIT "https://github.com/jcsda/fv3-jedi-linearmodel.git" BRANCH develop )
option(ENABLE_FV3_JEDI_DATA "Obtain fv3-jedi test data from fv3-jedi-data repository (vs tarball)" ON)
ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda/fv3-jedi.git" BRANCH develop )

# SOCA associated repositories
# TODO: Move the Icepack fork to EMC github
set(BUILD_ICEPACK "ON" CACHE STRING "Build the icepack library")
if ( BUILD_ICEPACK )
ecbuild_bundle( PROJECT icepack GIT "https://github.com/JCSDA-internal/Icepack.git" BRANCH feature/ecbuild-new )
endif()
ecbuild_bundle( PROJECT mom6 GIT "https://github.com/jcsda-internal/MOM6.git" BRANCH main-ecbuild RECURSIVE )
ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH develop )

# Build JEDI/DA or other peripherals
ecbuild_bundle( PROJECT gdas-utils SOURCE "./utils" )

# Build IODA converters
option(BUILD_IODA_CONVERTERS "Build IODA Converters" ON)
if(BUILD_IODA_CONVERTERS)
ecbuild_bundle( PROJECT iodaconv GIT "https://github.com/JCSDA-internal/ioda-converters.git" BRANCH develop )
endif()

# Land associated repositories
ecbuild_bundle( PROJECT land-imsproc GIT "https://github.com/NOAA-PSL/land-IMS_proc.git" TAG 6373819 )
ecbuild_bundle( PROJECT land-jediincr GIT "https://github.com/NOAA-PSL/land-apply_jedi_incr.git" TAG 2923344)

# ioda, ufo, fv3-jedi, and saber test data
#---------------------------------
if(CLONE_JCSDADATA)

# If IODA branch is being built set GIT_BRANCH_FUNC to IODA's current branch.
# If a tagged version of IODA is being built set GIT_TAG_FUNC to ioda's current tag. In this case,
# IODA test files will be download from UCAR DASH and ioda-data repo will not be cloned.
# When LOCAL_PATH_JEDI_TESTFILES is set to the directory of IODA test files stored
# in a local directory, ioda-data repo will not be cloned

find_branch_name(REPO_DIR_NAME ioda)
# When LOCAL_PATH_JEDI_TESTFILES is set to the directory of IODA test files stored
# in a local directory, ioda-data repo will not be cloned
if( NOT DEFINED ENV{LOCAL_PATH_JEDI_TESTFILES} AND NOT DEFINED GIT_TAG_FUNC )
ecbuild_bundle( PROJECT ioda-data GIT "https://github.com/JCSDA-internal/ioda-data.git" BRANCH develop )

# If IODA's current branch is available in ioda-data repo, that branch will be checked out
branch_checkout (REPO_DIR_NAME ioda-data
BRANCH ${GIT_BRANCH_FUNC} )
endif()

# same procedure for ufo-data
find_branch_name(REPO_DIR_NAME ufo)
if( NOT DEFINED ENV{LOCAL_PATH_JEDI_TESTFILES} AND NOT DEFINED GIT_TAG_FUNC )
ecbuild_bundle( PROJECT ufo-data GIT "https://github.com/JCSDA-internal/ufo-data.git" BRANCH develop )

# If UFO's current branch is available in ioda-data repo, that branch will be checked out
branch_checkout (REPO_DIR_NAME ufo-data
BRANCH ${GIT_BRANCH_FUNC} )
endif()

# same procedure for fv3-jedi-data
find_branch_name(REPO_DIR_NAME fv3-jedi)
if( NOT DEFINED ENV{LOCAL_PATH_JEDI_TESTFILES} AND NOT DEFINED GIT_TAG_FUNC )
ecbuild_bundle( PROJECT fv3-jedi-data GIT "https://github.com/JCSDA-internal/fv3-jedi-data.git" BRANCH develop )

# If fv3-jedi's current branch is available in ioda-data repo, that branch will be checked out
branch_checkout (REPO_DIR_NAME fv3-jedi-data
BRANCH ${GIT_BRANCH_FUNC} )
endif()

endif(CLONE_JCSDADATA)

endif(BUILD_GDASBUNDLE)

# Install utility scripts.
add_subdirectory(ush)

# Include testing.
add_subdirectory(test)

# Finalize bundle
# ---------------
ecbuild_bundle_finalize()
4 changes: 2 additions & 2 deletions build.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ while getopts "p:t:c:hvdfa" opt; do
done

case ${BUILD_TARGET} in
hera | orion)
hera | orion | hercules)
echo "Building GDASApp on $BUILD_TARGET"
source $dir_root/ush/module-setup.sh
module use $dir_root/modulefiles
Expand Down Expand Up @@ -113,7 +113,7 @@ echo "Configuring ..."
set -x
cmake \
${CMAKE_OPTS:-} \
$dir_root
$dir_root/sorc
set +x

# Build
Expand Down
32 changes: 8 additions & 24 deletions ci/stable_driver.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,19 +64,10 @@ cd $stableroot/$datestr/global-workflow/sorc/gdas.cd
git checkout develop
git pull

# ==============================================================================
# run ecbuild to get the repos cloned
mkdir -p build

cd build
ecbuild ../
cd ..
rm -rf build

# ==============================================================================
# update the hashes to the most recent
gdasdir=$stableroot/$datestr/global-workflow/sorc/gdas.cd
$my_dir/stable_mark.sh $gdasdir
$my_dir/../ush/submodules/update_develop.sh $gdasdir

# ==============================================================================
# run the automated testing
Expand All @@ -85,40 +76,33 @@ ci_status=$?
total=0
if [ $ci_status -eq 0 ]; then
cd $gdasdir
# copy the CMakeLists file for safe keeping
cp $gdasdir/CMakeLists.txt $gdasdir/CMakeLists.txt.new
total=$(($total+$?))
if [ $total -ne 0 ]; then
echo "Unable to cp CMakeLists" >> $stableroot/$datestr/output
fi
# checkout feature/stable-nightly
git stash
total=$(($total+$?))
if [ $total -ne 0 ]; then
echo "Unable to cp CMakeLists" >> $stableroot/$datestr/output
echo "Unable to git stash" >> $stableroot/$datestr/output
fi
git checkout feature/stable-nightly
total=$(($total+$?))
if [ $total -ne 0 ]; then
echo "Unable to cp CMakeLists" >> $stableroot/$datestr/output
echo "Unable to checkout feature/stable-nightly" >> $stableroot/$datestr/output
fi
# merge in develop
git merge develop
total=$(($total+$?))
if [ $total -ne 0 ]; then
echo "Unable to merge develop" >> $stableroot/$datestr/output
fi
# force move the copy to the original path of CMakeLists.txt
/bin/mv -f $gdasdir/CMakeLists.txt.new $gdasdir/CMakeLists.txt
# add in submodules
git stash pop
total=$(($total+$?))
if [ $total -ne 0 ]; then
echo "Unable to mv CMakeLists" >> $stableroot/$datestr/output
echo "Unable to git stash pop" >> $stableroot/$datestr/output
fi
# commit this change and push
git add CMakeLists.txt
$my_dir/../ush/submodules/add_submodules.sh $gdasdir
total=$(($total+$?))
if [ $total -ne 0 ]; then
echo "Unable to add CMakeLists to commit" >> $stableroot/$datestr/output
echo "Unable to add updated submodules to commit" >> $stableroot/$datestr/output
fi
git diff-index --quiet HEAD || git commit -m "Update to new stable build on $datestr"
total=$(($total+$?))
Expand Down
Loading

0 comments on commit de86fb4

Please sign in to comment.