Merge pull request #1274 from JCSDA-internal/release/1.1.0

Release/1.1.0
JCSDA · Jun 11, 2021 · 2af9b91 · 2af9b91
2 parents b1804f5 + be17f9d
commit 2af9b91
Show file tree

Hide file tree

Showing 1,123 changed files with 101,674 additions and 13,106 deletions.
diff --git a/.gitattributes b/.gitattributes
@@ -1,2 +1,3 @@
 *.nc filter=lfs diff=lfs merge=lfs -text
 *.nc4 filter=lfs diff=lfs merge=lfs -text
+*.dat filter=lfs diff=lfs merge=lfs -text
diff --git a/.github/ISSUE_TEMPLATE/basic-issue.md b/.github/ISSUE_TEMPLATE/basic-issue.md
@@ -0,0 +1,10 @@
+---
+name: Regular issue
+about: This is a blank issue.
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+
diff --git a/.github/ISSUE_TEMPLATE/missing-docs.md b/.github/ISSUE_TEMPLATE/missing-docs.md
@@ -0,0 +1,30 @@
+---
+name: 'Missing Documentation'
+about: 'Use this issue to note missing documentation on ReadTheDocs and/or Doxygen'
+title: "[Documentation]: Give a descriptive title here"
+labels: documentation
+assignees: huishao-r, rhoneyager, mmiesch
+
+---
+
+## Description
+
+> Edit the text of this pull request to describe what is missing. You should include the file name or the class name.
+
+
+
+## ZenHub organization
+
+Please make sure that you have properly set:
+- Labels: documentation
+- Milestone: OBS - current month
+- Estimate: usually 0.5 - 1
+- Epics: Select a relevant epic. Here are a few hints.
+    - JEDI1.10: For **generic** UFO components and bias correction. Generic means a core UFO component, such as a base class
+        or a component of JEDI that more than one agency will use.
+    - OBS3.1.2: For **specific** ObsOperators, Filters, and ObsFunctions.
+- Assignees: Generally, the person who created the code is responsible for writing the documentation.
+    The "git blame" feature may be quite helpful in determining this. @huishao-r, @mmiesch, and @rhoneyager are
+    automatically added to help track these issues.
+
+
diff --git a/CI/CMakeLists.txt b/CI/CMakeLists.txt
@@ -10,7 +10,7 @@
 
 cmake_minimum_required( VERSION 3.12 FATAL_ERROR )
 
-project( ufo-bundle LANGUAGES C CXX Fortran )
+project( ufo-bundle VERSION 1.0.0 LANGUAGES C CXX Fortran )
 
 find_package(ecbuild)
 set( CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake;${CMAKE_CURRENT_SOURCE_DIR}/ecbuild/cmake;${CMAKE_MODULE_PATH}")
@@ -19,20 +19,45 @@ include( ecbuild_bundle )
 set( ENABLE_MPI ON CACHE BOOL "Compile with MPI" )
 
 ecbuild_bundle_initialize()
-ecbuild_bundle( PROJECT jedicmake GIT "https://github.com/JCSDA/jedi-cmake.git" )
+ecbuild_bundle( PROJECT jedicmake GIT "https://github.com/JCSDA-internal/jedi-cmake.git" )
+include( jedicmake/cmake/Functions/git_functions.cmake )
 
 option("BUNDLE_SKIP_ECKIT" "Don't build eckit" "ON") #Skip eckit build unless user passes -DBUNDLE_SKIP_ECKIT=OFF
-ecbuild_bundle( PROJECT eckit GIT "https://github.com/JCSDA/eckit.git" )
-ecbuild_bundle( PROJECT fckit GIT "https://github.com/JCSDA/fckit.git" )
-ecbuild_bundle( PROJECT atlas GIT "https://github.com/JCSDA/atlas.git" )
+option("BUNDLE_SKIP_FCKIT" "Don't build fckit" "ON") #Skip fckit build unless user passes -DBUNDLE_SKIP_FCKIT=OFF
+option("BUNDLE_SKIP_ATLAS" "Don't build atlas" "ON") #Skip atlas build unless user passes -DBUNDLE_SKIP_ATLAS=OFF
 
-ecbuild_bundle( PROJECT oops GIT "https://github.com/JCSDA/oops.git" )
-ecbuild_bundle( PROJECT saber GIT "https://github.com/JCSDA/saber.git" )
-#ecbuild_bundle( PROJECT gsw  GIT "https://github.com/JCSDA/GSW-Fortran.git" BRANCH develop  UPDATE )
-ecbuild_bundle( PROJECT crtm GIT "https://github.com/JCSDA/crtm.git" )
-ecbuild_bundle( PROJECT ioda GIT "https://github.com/JCSDA/ioda.git" )
-ecbuild_bundle( PROJECT ufo  GIT "https://github.com/JCSDA/ufo.git" )
+ecbuild_bundle( PROJECT eckit GIT "https://github.com/ecmwf/eckit.git" TAG 1.16.0 )
+ecbuild_bundle( PROJECT fckit GIT "https://github.com/ecmwf/fckit.git" TAG 0.9.2 )
+ecbuild_bundle( PROJECT atlas GIT "https://github.com/ecmwf/atlas.git" TAG 0.24.1 )
 
+ecbuild_bundle( PROJECT oops GIT "https://github.com/JCSDA-internal/oops.git" )
+ecbuild_bundle( PROJECT saber GIT "https://github.com/JCSDA-internal/saber.git" )
+#ecbuild_bundle( PROJECT gsw  GIT "https://github.com/JCSDA-internal/GSW-Fortran.git" )
+ecbuild_bundle( PROJECT crtm GIT "https://github.com/JCSDA-internal/crtm.git" )
+ecbuild_bundle( PROJECT ioda GIT "https://github.com/JCSDA-internal/ioda.git" )
+ecbuild_bundle( PROJECT rttov  GIT "https://github.com/JCSDA-internal/rttov.git" )
+ecbuild_bundle( PROJECT ropp-ufo  GIT "https://github.com/JCSDA-internal/ropp-test.git" )
+ecbuild_bundle( PROJECT geos-aero  GIT "https://github.com/JCSDA-internal/geos-aero.git" )
+ecbuild_bundle( PROJECT ufo  GIT "https://github.com/JCSDA-internal/ufo.git" )
+
+# Don't need ioda-data in CI because we are only testing ufo
+
+# If IODA branch is being built set GIT_BRANCH_FUNC to IODA's current branch.
+# If a tagged version of IODA is being built set GIT_TAG_FUNC to ioda's current tag. In this case,
+# IODA test files will be download from UCAR DASH and ioda-data repo will not be cloned.
+#find_branch_name(REPO_DIR_NAME ioda)
+# When LOCAL_PATH_JEDI_TESTFILES is set to the directory of IODA test files stored
+# in a local directory, ioda-data repo will not be cloned
+
+#if( NOT DEFINED ENV{LOCAL_PATH_JEDI_TESTFILES} AND NOT DEFINED ${GIT_TAG_FUNC} )
+#  ecbuild_bundle( PROJECT ioda-data GIT "https://github.com/JCSDA-internal/ioda-data.git" )
+#endif()
+
+# same procedure for ufo-data
+find_branch_name(REPO_DIR_NAME ufo)
+if( NOT DEFINED ENV{LOCAL_PATH_JEDI_TESTFILES} AND NOT DEFINED ${GIT_TAG_FUNC} )
+  ecbuild_bundle( PROJECT ufo-data GIT "https://github.com/JCSDA-internal/ufo-data.git" )
+endif()
 
 # Build Doxygen documentation
 option(BUILD_UFO_BUNDLE_DOC "Build documentation" OFF)
@@ -44,4 +69,3 @@ ecbuild_bundle_finalize()
 
 include(cmake/cdash-integration.cmake)
 include(CTest)
-
diff --git a/CI/buildspec_clang.yml b/CI/buildspec_clang.yml
@@ -1,6 +1,7 @@
 version: 0.2
 
 env:
+  shell: bash
   parameter-store:
     GIT_USER: "/CodeBuild/Git_USER"
     GIT_PASS: "/CodeBuild/Git_PASS"
@@ -45,7 +46,7 @@ phases:
       - git lfs install # creates .gitconfig
 
       #  Set crtm branch name
-      - export CRTM_JEDI_BRANCH="v2.3-jedi"
+      - export CRTM_JEDI_BRANCH="release/crtm_jedi"
 
       - if [ "$CODEBUILD_GIT_BRANCH" = "develop" ];
         then export CODEBUILD_GIT_BRANCH_FORK="release-stable";
@@ -55,29 +56,26 @@ phases:
         echo "CODEBUILD_GIT_BRANCH_FORK=${CODEBUILD_GIT_BRANCH_FORK}";
         fi
 
+      # Upload branch name and commit sha as CodeBuild artifact to S3
+      - mkdir -p /jcsda/artifacts
+      - echo ${CODEBUILD_GIT_BRANCH} > /jcsda/artifacts/branch_name.txt
+      - echo ${CODEBUILD_RESOLVED_SOURCE_VERSION} > /jcsda/artifacts/commit_sha.txt
+
       ##  public repos
       # ufo
-      - ./clone.sh $GIT_USER $GIT_PASS jcsda/ufo $CODEBUILD_GIT_BRANCH ufo /jcsda/ufo-bundle develop
+      - ./clone.sh $GIT_USER $GIT_PASS jcsda-internal/ufo $CODEBUILD_GIT_BRANCH ufo /jcsda/ufo-bundle develop
 
       # crtm
-      - ./clone.sh $GIT_USER $GIT_PASS jcsda/crtm $CODEBUILD_GIT_BRANCH_CRTM crtm /jcsda/ufo-bundle $CRTM_JEDI_BRANCH
+      - ./clone.sh $GIT_USER $GIT_PASS jcsda-internal/crtm $CODEBUILD_GIT_BRANCH_CRTM crtm /jcsda/ufo-bundle $CRTM_JEDI_BRANCH
 
       # ioda
-      - ./clone.sh $GIT_USER $GIT_PASS jcsda/ioda $CODEBUILD_GIT_BRANCH ioda /jcsda/ufo-bundle develop
+      - ./clone.sh $GIT_USER $GIT_PASS jcsda-internal/ioda $CODEBUILD_GIT_BRANCH ioda /jcsda/ufo-bundle develop
 
       # saber
-      - ./clone.sh $GIT_USER $GIT_PASS jcsda/saber $CODEBUILD_GIT_BRANCH saber /jcsda/ufo-bundle develop
+      - ./clone.sh $GIT_USER $GIT_PASS jcsda-internal/saber $CODEBUILD_GIT_BRANCH saber /jcsda/ufo-bundle develop
 
       # oops
-      - ./clone.sh $GIT_USER $GIT_PASS jcsda/oops $CODEBUILD_GIT_BRANCH oops /jcsda/ufo-bundle develop
-
-
-      ##  ecmwf repos
-      # atlas
-      - ./clone.sh $GIT_USER $GIT_PASS jcsda-internal/atlas $CODEBUILD_GIT_BRANCH_FORK atlas /jcsda/ufo-bundle release-stable
-
-      #fckit
-      - ./clone.sh $GIT_USER $GIT_PASS jcsda-internal/fckit $CODEBUILD_GIT_BRANCH_FORK fckit /jcsda/ufo-bundle release-stable
+      - ./clone.sh $GIT_USER $GIT_PASS jcsda-internal/oops $CODEBUILD_GIT_BRANCH oops /jcsda/ufo-bundle develop
 
 
       ##  internal repos
@@ -99,33 +97,33 @@ phases:
       #gsw
       - ./clone.sh $GIT_USER $GIT_PASS jcsda-internal/GSW-Fortran $CODEBUILD_GIT_BRANCH gsw /jcsda/ufo-bundle develop
 
+      # ufo-test
+      - ./clone.sh $GIT_USER $GIT_PASS jcsda-internal/ufo-data $CODEBUILD_GIT_BRANCH ufo-data /jcsda/ufo-bundle develop
+
       # move CMakeLists.txt
       - cp CMakeLists.txt /jcsda/ufo-bundle
       - cp -r cmake /jcsda/ufo-bundle/
-      - cd /jcsda/ufo-bundle
+      - cp /jcsda/ufo-bundle/cmake/CTestConfig.cmake /jcsda/ufo-bundle
 
-      - ls
       - cd /jcsda/ufo-bundle
       - ls
 
-      - sed -i 's/BRANCH develop  UPDATE//' CMakeLists.txt
-      - sed -i 's/BRANCH develop //' CMakeLists.txt
-      - sed -i 's/BRANCH release-stable UPDATE//' CMakeLists.txt
-      - sed -i -e '/ecbuild_bundle( PROJECT ufo/i\
-        ecbuild_bundle( PROJECT ropp-ufo  GIT "https://github.com/JCSDA/ropp-test.git" )' CMakeLists.txt
-      - sed -i -e '/ecbuild_bundle( PROJECT ufo/i\
-        ecbuild_bundle( PROJECT geos-aero  GIT "https://github.com/JCSDA/geos-aero.git" )' CMakeLists.txt
-
   build:
+    on-failure: CONTINUE
     commands:
       - echo Executing build phase
       - echo $CODEBUILD_BUILD_SUCCEEDING
       - export BUILD_STATUS="0"
       - echo $BUILD_STATUS
 
+      # configure and build
       - cd /build_container
-      - ecbuild -DBUILD_RTTOV=1 /jcsda/ufo-bundle/
+      - ecbuild -Wno-dev -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCDASH_OVERRIDE_GIT_BRANCH=$CODEBUILD_GIT_BRANCH -DCTEST_UPDATE_VERSION_ONLY=FALSE /jcsda/ufo-bundle
       - cd ufo
+      - cp ../DartConfiguration.tcl .
+      - sed -i 's/ufo-bundle/ufo-bundle\/ufo/' DartConfiguration.tcl
+      - sed -i 's/build_container/build_container\/ufo/' DartConfiguration.tcl
+      - cat DartConfiguration.tcl
       - make -j4
 
       - if [ "$CODEBUILD_BUILD_SUCCEEDING" = "1" ];
@@ -134,16 +132,32 @@ phases:
         fi
       - echo $BUILD_STATUS
 
+      # run ctest
+      - cd /build_container/ufo
+      - ctest -C RelWithDebInfo -D ExperimentalTest
+
+    finally:
       - cd /build_container/ufo
-      - ctest
+      - ctest -C RelWithDebInfo -D ExperimentalSubmit -M Continuous -- --track Continuous --group Continuous
 
   post_build:
     commands:
       - echo Executing post_build phase
       - echo $CODEBUILD_BUILD_SUCCEEDING
-      - if [ "$BUILD_STATUS" = "1" ] && [ "$CODEBUILD_BUILD_SUCCEEDING" = "0" ];
-        then echo "Build passed, rerunning failed tests";
-        cd /build_container/ufo;
-        ctest -VV --rerun-failed;
+      - echo $BUILD_STATUS
+
+      # upload find cdash url and upload it as CodeBuild artifact to S3
+      - if [ "$BUILD_STATUS" = "1" ];
+        then echo "Build & tests passed, find cdash url";
+        bash /jcsda/ufo-bundle/ufo/CI/cdash-url.sh /build_container/ufo/Testing;
+        url=$(bash /jcsda/ufo-bundle/ufo/CI/cdash-url.sh /build_container/ufo/Testing);
+        echo $url;
+        echo ${url} > /jcsda/artifacts/cdash-url.txt;
+        cat /jcsda/artifacts/cdash-url.txt;
         else echo "Build failed";
         fi
+
+artifacts:
+  files:
+    - '/jcsda/artifacts/*'
+  name: ufo-clang-url