Skip to content

Commit

Permalink
Merge pull request #1241 from JCSDA-internal/release/1.1.0
Browse files Browse the repository at this point in the history
Release/1.1.0
  • Loading branch information
Mark Miesch committed Jun 11, 2021
2 parents 40e85e4 + f04e906 commit 0a07af4
Show file tree
Hide file tree
Showing 736 changed files with 19,711 additions and 11,730 deletions.
7 changes: 7 additions & 0 deletions .codecov.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,13 @@ coverage:
precision: 2
round: down
range: "70...100"
status:
project:
default:
threshold: 0.05%
patch:
default:
threshold: 2%

ignore:
- "src/test"
Expand Down
28 changes: 28 additions & 0 deletions .github/ISSUE_TEMPLATE/bug-report.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
---
name: Bug report
about: Use this template to report bugs
title: "[Bug]"
labels: 'bug'
assignees: ''

---

## Current behavior (describe the bug)
>[Be sure to add a Pipeline, Label, Estimate, Assignees, and Epic](https://jointcenterforsatellitedataassimilation-jedi-docs.readthedocs-hosted.com/en/latest/inside/practices/issues.html)
## To Reproduce

> What computer are you running on?
> What compilers/modules are you using?
> Steps to reproduce the behavior
1.
2.
3.
...

## Expected behavior

## Additional information (optional)
8 changes: 8 additions & 0 deletions .github/ISSUE_TEMPLATE/config.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
blank_issues_enabled: false
contact_links:
- name: JCSDA
url: https://jcsda.org/
about: JCSDA web site
- name: Forums
url: https://forums.jcsda.org/
about: JCSDA user/developer forums
26 changes: 26 additions & 0 deletions .github/ISSUE_TEMPLATE/general-issue.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
---
name: General issue template
about: Use this template for general issues
title: "[New issue]"
labels: ''
assignees: ''

---

## Description
>Provide a detailed description of this issue.
>What problem needs to be fixed? What new capability needs to be added?
>If this is a bug, describe the current behavior (or use the bug template).
>[Be sure to add a Pipeline, Label, Estimate, Assignees, and Epic](https://jointcenterforsatellitedataassimilation-jedi-docs.readthedocs-hosted.com/en/latest/inside/practices/issues.html)
## Requirements

>If this is a new feature: What does the new code need to accomplish? Does it require new software dependencies (e.g. new jedi-stack components or new python modules?)
>If this is a bugfix: What is the expected behavior?
## Acceptance Criteria (Definition of Done)
>What does it mean for this to be finished?
## Dependencies
>What must be done before this can be done? Add issue dependencies in ZenHub as appropriate
>Does this block progress on other issues? Add this issue as a dependency to other ZenHub issues as appropriate
22 changes: 9 additions & 13 deletions CI/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@

# (C) Copyright 2017 UCAR
#
# This software is licensed under the terms of the Apache Licence Version 2.0
Expand All @@ -8,25 +8,21 @@
# OOPS bundle
#

project( oops-bundle C CXX Fortran )
cmake_minimum_required( VERSION 3.12 FATAL_ERROR )

cmake_minimum_required( VERSION 3.3.2 FATAL_ERROR )
project( oops-bundle VERSION 1.0.0 LANGUAGES C CXX Fortran )

set( CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake;${CMAKE_CURRENT_SOURCE_DIR}/ecbuild/cmake;${CMAKE_MODULE_PATH}")
find_package(ecbuild REQUIRED)
list( APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake)
include( ecbuild_bundle )

set( ENABLE_MPI ON CACHE BOOL "Compile with MPI" )

ecbuild_bundle_initialize()

ecbuild_requires_macro_version( 2.7 )

ecbuild_add_option( FEATURE OMP
DEFAULT ON
DESCRIPTION "Use OpenMP" )

ecbuild_bundle( PROJECT fckit GIT "https://github.com/JCSDA/fckit.git" )
ecbuild_bundle( PROJECT atlas GIT "https://github.com/JCSDA/atlas.git" )
ecbuild_bundle( PROJECT oops GIT "https://github.com/JCSDA/oops.git" )
ecbuild_bundle( PROJECT oops GIT "https://github.com/jcsda-internal/oops.git" )

ecbuild_bundle_finalize()

include(cmake/cdash-integration.cmake)
include(CTest)
44 changes: 36 additions & 8 deletions CI/buildspec_clang.yml
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
version: 0.2

env:
shell: bash
parameter-store:
GIT_USER: "/CodeBuild/Git_USER"
GIT_PASS: "/CodeBuild/Git_PASS"
Expand All @@ -21,7 +22,15 @@ phases:
- echo $CODEBUILD_WEBHOOK_TRIGGER
- echo $CODEBUILD_WEBHOOK_BASE_REF

- if [ "$CODEBUILD_WEBHOOK_EVENT" = "PULL_REQUEST_MERGED" ];
- gh_source=$(pwd)
- echo ${gh_source}
- echo $CODEBUILD_SRC_DIR

# Codebuild only runs on PUSH events if HEAD_REF
# is refs/heads/develop (merge to develop). In this
# case CODEBUILD_GIT_BRANCH="develop"

- if [ "$CODEBUILD_WEBHOOK_EVENT" = "PUSH" ];
then export CODEBUILD_GIT_BRANCH="develop";
echo "Merging to develop";
else export CODEBUILD_GIT_BRANCH=${CODEBUILD_WEBHOOK_HEAD_REF#refs/heads/};
Expand All @@ -31,9 +40,9 @@ phases:
- echo "CODEBUILD_SOURCE_VERSION=${CODEBUILD_SOURCE_VERSION}"

- echo "check for same branch names, except for develop"
- if ! [ "$CODEBUILD_GIT_BRANCH" = "develop" ]; then
- if ! [ "$CODEBUILD_GIT_BRANCH" = "develop" ]; then
pip install boto3;
pip install pygithub;
pip install pygithub;
python CI/update_webhook_branchname.py $CODEBUILD_GIT_BRANCH $CODEBUILD_RESOLVED_SOURCE_VERSION;
fi

Expand All @@ -45,15 +54,34 @@ phases:
- echo ${CODEBUILD_RESOLVED_SOURCE_VERSION} > artifacts/commit_sha.txt

# get jedi-build-package
- git clone https://$GIT_USER:[email protected]/jcsda-internal/jedi-build-package
- git clone -b feature/ecbuild35 https://$GIT_USER:[email protected]/jcsda-internal/jedi-build-package

- cd jedi-build-package
- pip install --user -e .

- ~/.local/bin/jedi-build --gh-token=$GIT_PASS -j 4 --env-id=docker-clang --branch-map atlas:release-stable fckit:release-stable -br ${CODEBUILD_GIT_BRANCH} -p oops -vvv -3 -u --submit-dashboard --site CodeBuild --abort-on-build-errors --abort-on-test-errors
- ~/.local/bin/jedi-build --gh-token=$GIT_PASS -j 4 --env-id=docker-clang --branch-map atlas:release-stable fckit:release-stable -br ${CODEBUILD_GIT_BRANCH} -p oops -vvv -3 -u --submit-dashboard --site CodeBuild --abort-on-build-errors --abort-on-test-errors --cleanup=False 2>&1 | tee jedi_build_output.txt ; test ${PIPESTATUS[0]} -eq 0

#find CDASH URL
- Done_path_line=$(grep 'Done.xml' jedi_build_output.txt | tail -1)
- echo $Done_path_line
- Done_path=$(echo ${Done_path_line} | sed 's/Uploaded://')
- echo $Done_path
- cat $Done_path
# retrieve buildID from Done.xml
- buildID=$(cat $Done_path | grep -o -P '(?<=buildId>).*(?=</build)')
- echo ${buildID}
- url=https://cdash.jcsda.org/buildSummary.php?buildid=${buildID}
- echo ${url}

#create artifacts
- mkdir -p /jcsda/artifacts/
- echo ${url} > /jcsda/artifacts/cdash-url.txt
- cat /jcsda/artifacts/cdash-url.txt
- echo ${CODEBUILD_GIT_BRANCH} > /jcsda/artifacts/branch_name.txt
- echo ${CODEBUILD_RESOLVED_SOURCE_VERSION} > /jcsda/artifacts/commit_sha.txt
- ls /jcsda/artifacts/


artifacts:
files:
- 'artifacts/*'
name: artifact-oops-clang
- '/jcsda/artifacts/*'
name: oops-clang-url
130 changes: 71 additions & 59 deletions CI/buildspec_gnu.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ phases:

- echo $CODEBUILD_RESOLVED_SOURCE_VERSION
- echo $CODEBUILD_SOURCE_REPO_URL
- org_name=$(echo $CODEBUILD_SOURCE_REPO_URL | awk '{split($0,org,"/"); print org[4]}')
- echo $org_name
- echo $CODEBUILD_SOURCE_VERSION

- echo $CODEBUILD_WEBHOOK_MERGE_COMMIT
Expand All @@ -22,7 +24,11 @@ phases:
- echo $CODEBUILD_WEBHOOK_TRIGGER
- echo $CODEBUILD_WEBHOOK_BASE_REF

- if [ "$CODEBUILD_WEBHOOK_EVENT" = "PULL_REQUEST_MERGED" ];
# Codebuild only runs on PUSH events if HEAD_REF
# is refs/heads/develop (merge to develop). In this
# case CODEBUILD_GIT_BRANCH="develop"

- if [ "$CODEBUILD_WEBHOOK_EVENT" = "PUSH" ];
then export CODEBUILD_GIT_BRANCH="develop";
echo "Merging to develop";
else export CODEBUILD_GIT_BRANCH=${CODEBUILD_WEBHOOK_HEAD_REF#refs/heads/};
Expand All @@ -31,102 +37,108 @@ phases:
- echo "CODEBUILD_GIT_BRANCH=${CODEBUILD_GIT_BRANCH}"
- echo "CODEBUILD_SOURCE_VERSION=${CODEBUILD_SOURCE_VERSION}"

- echo MPI setup for Docker
- mkdir -p /var/run/sshd
- ssh-keygen -A
- sed -i 's/#PermitRootLogin yes/PermitRootLogin yes/g' /etc/ssh/sshd_config
- sed -i 's/#RSAAuthentication yes/RSAAuthentication yes/g' /etc/ssh/sshd_config
- sed -i 's/#PubkeyAuthentication yes/PubkeyAuthentication yes/g' /etc/ssh/sshd_config

- groupadd jcsda -g 9999
- useradd jcsdauser
- mkdir -p /jcsda /build_container
- chown -R jcsdauser:jcsda /build_container /usr/local
- chmod 6755 /jcsda /build_container /usr/local

- mkdir /jcsda/.ssh ; echo "StrictHostKeyChecking no" > /jcsda/.ssh/config
- mkdir -p /jcsda/.openmpi
- mkdir -p /jcsda/oops-bundle
- mkdir -p /home/jcsdauser/.openmpi

- cp CI/default-mca-params.conf /home/jcsdauser/.openmpi/mca-params.conf
- cat /home/jcsdauser/.openmpi/mca-params.conf
- chown -R jcsdauser:jcsda /jcsda/

- su - jcsdauser -c "ssh-keygen -f /jcsda/.ssh/id_rsa -t rsa -N ''
&& chmod 600 /jcsda/.ssh/config
&& chmod 700 /jcsda/.ssh
&& cp /jcsda/.ssh/id_rsa.pub /jcsda/.ssh/authorized_keys
&& echo MPI setup for Docker done"

pre_build:
commands:
- echo Executing pre_build phase
- mkdir -p /jcsda/oops-bundle

- git lfs install
- cp ~/.gitconfig /home/jcsdauser/
- cp CI/CMakeLists.txt /jcsda/oops-bundle

- cp ~/.gitconfig /home/jedi/
- cd CI

- if [ "$CODEBUILD_GIT_BRANCH" = "develop" ];
then export CODEBUILD_GIT_BRANCH_FORK="release-stable";
else export CODEBUILD_GIT_BRANCH_FORK=${CODEBUILD_GIT_BRANCH};
echo "CODEBUILD_GIT_BRANCH_FORK=${CODEBUILD_GIT_BRANCH_FORK}";
fi

# Upload branch name and commit sha as CodeBuild artifact to S3
- mkdir /jcsda/artifacts
- echo ${CODEBUILD_GIT_BRANCH} > /jcsda/artifacts/branch_name.txt
- echo ${CODEBUILD_RESOLVED_SOURCE_VERSION} > /jcsda/artifacts/commit_sha.txt

# oops
- ./clone.sh $GIT_USER $GIT_PASS jcsda/oops $CODEBUILD_GIT_BRANCH oops /jcsda/oops-bundle develop
# fckit
- ./clone.sh $GIT_USER $GIT_PASS jcsda-internal/fckit $CODEBUILD_GIT_BRANCH_FORK fckit /jcsda/oops-bundle release-stable
# atlas
- ./clone.sh $GIT_USER $GIT_PASS jcsda-internal/atlas $CODEBUILD_GIT_BRANCH_FORK atlas /jcsda/oops-bundle release-stable
- ./clone.sh $GIT_USER $GIT_PASS jcsda-internal/oops $CODEBUILD_GIT_BRANCH oops /jcsda/oops-bundle develop

# move CMakeLists.txt from oops/CI to bundle directory
- cp CMakeLists.txt /jcsda/oops-bundle/CMakeLists.txt

# cdash upload setup
- mkdir /jcsda/oops-bundle/cmake
- cp cmake/CTestCustom.ctest.in /jcsda/oops-bundle/cmake/
- cp cmake/cdash-integration.cmake /jcsda/oops-bundle/cmake/
- cp cmake/CTestConfig.cmake /jcsda/oops-bundle/

- cd /jcsda/oops-bundle
- ls

build:
on-failure: CONTINUE
commands:
- echo Executing build phase
- su - jcsdauser -c "cd /build_container
- export BUILD_STATUS="0"
- echo $BUILD_STATUS
- echo $CODEBUILD_BUILD_SUCCEEDING

# configure and build
- su - jedi -c "cd /home/jedi
&& ls
&& export FC=mpifort
&& export CC=mpicc
&& export CXX=mpicxx
&& cmake -DCMAKE_MODULE_PATH=/usr/local/share/ecbuild/cmake/ -DCMAKE_BUILD_TYPE=Debug -DENABLE_GPROF=ON -DCMAKE_CXX_COMPILER=$(which g++) -DCMAKE_C_COMPILER=$(which gcc) -DCMAKE_Fortran_COMPILER=$(which gfortran) /jcsda/oops-bundle/
&& cd oops
&& make -j4"
&& ecbuild -Wno-dev -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCDASH_OVERRIDE_GIT_BRANCH=$CODEBUILD_GIT_BRANCH -DCTEST_UPDATE_VERSION_ONLY=FALSE -DENABLE_GPROF=ON /jcsda/oops-bundle/"

- export BUILD_STATUS="0"
- echo $BUILD_STATUS
- echo $CODEBUILD_BUILD_SUCCEEDING
- su - jedi -c "cd /home/jedi/oops
&& export FC=mpifort
&& export CC=mpicc
&& export CXX=mpicxx
&& cp ../DartConfiguration.tcl .
&& sed -i 's/oops-bundle/oops-bundle\/oops/' DartConfiguration.tcl
&& sed -i 's/jedi/jedi\/oops/' DartConfiguration.tcl
&& cat DartConfiguration.tcl
&& make -j4"

- if [ "$CODEBUILD_BUILD_SUCCEEDING" = "1" ];
then export BUILD_STATUS="1";
echo "Build passed";
fi
- echo $BUILD_STATUS

- su - jcsdauser -c "cd /build_container
&& ls
# run ctest and upload to cdash
- su - jedi -c "cd /home/jedi/oops
&& export FC=mpifort
&& export CC=mpicc
&& export CXX=mpicxx
&& ctest -C RelWithDebInfo -D ExperimentalTest -E test_qg_4dvar_saddlepoint"

finally:
- su - jedi -c "cd /home/jedi/oops
&& export FC=mpifort
&& export CC=mpicc
&& export CXX=mpicxx
&& cd oops
&& ctest"

&& ctest -C RelWithDebInfo -D ExperimentalSubmit -M Continuous -- --track Continuous --group Continuous"

post_build:
commands:
- echo Executing post_build phase
- echo $CODEBUILD_BUILD_SUCCEEDING
- if [ "$BUILD_STATUS" = "1" ] && [ "$CODEBUILD_BUILD_SUCCEEDING" = "0" ];
then echo "Build passed, rerun failed tests";
su - jcsdauser -c "cd /build_container/oops
&& ctest -VV --rerun-failed";
- echo $BUILD_STATUS

# upload find cdash url and upload it as CodeBuild artifact to S3
- if [ "$BUILD_STATUS" = "1" ];
then echo "Build & tests passed, find cdash url";
bash /jcsda/oops-bundle/oops/CI/cdash-url.sh /home/jedi/oops/Testing;
url=$(bash /jcsda/oops-bundle/oops/CI/cdash-url.sh /home/jedi/oops/Testing);
echo $url;
echo ${url} > /jcsda/artifacts/cdash-url.txt;
cat /jcsda/artifacts/cdash-url.txt;
else echo "Build failed";
fi

- echo 'Connect to CodeCov'
- cd /build_container/oops
- cd /home/jedi/oops
- pwd
- ls
- bash /jcsda/oops-bundle/oops/CI/codecov_script.sh
- bash /jcsda/oops-bundle/oops/CI/codecov_script_$org_name.sh

artifacts:
files:
- '/jcsda/artifacts/*'
name: oops-gnu-url
Loading

0 comments on commit 0a07af4

Please sign in to comment.