From e510bd7d7b51780eac225188b32ea71d9ee53607 Mon Sep 17 00:00:00 2001 From: Boris Acha <1165516@us.ray.com> Date: Fri, 19 Aug 2022 10:36:01 -0400 Subject: [PATCH 1/4] New data container jenkinsfile --- Jenkinsfile | 173 +++++++++++++++++++++++++++++++--------------------- 1 file changed, 105 insertions(+), 68 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 25c948f..e746da7 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,103 +1,140 @@ pipeline { - agent { - label 'docker' - } - - environment { - DOCKER_HUB = credentials('DockerHubNOAAEPIC') - UBUNTU_GNU_IMAGE_NAME = 'ubuntu20.04-gnu9.3' - UBUNTU_GNU_IMAGE_VERSION = '0.1' - UBUNTU_GNU_IMAGE_TAG = "${env.DOCKER_HUB_USR}/${env.UBUNTU_GNU_IMAGE_NAME}:${env.UBUNTU_GNU_IMAGE_VERSION}" - UBUNTU_GNU_HPCSTACK_IMAGE_NAME = "${env.UBUNTU_GNU_IMAGE_NAME}-hpc-stack" - UBUNTU_GNU_HPCSTACK_IMAGE_VERSION = '0.1' - UBUNTU_GNU_HPCSTACK_IMAGE_TAG = "${env.DOCKER_HUB_USR}/${env.UBUNTU_GNU_HPCSTACK_IMAGE_NAME}:${env.UBUNTU_GNU_HPCSTACK_IMAGE_VERSION}" - UBUNTU_GNU_SRW_IMAGE_NAME = "${env.UBUNTU_GNU_IMAGE_NAME}-ufs-srwapp" - UBUNTU_GNU_SRW_IMAGE_VERSION = '0.1' - UBUNTU_GNU_SRW_IMAGE_TAG = "${env.DOCKER_HUB_USR}/${env.UBUNTU_GNU_SRW_IMAGE_NAME}:${env.UBUNTU_GNU_SRW_IMAGE_VERSION}" - } - - stages { - stage('Prepare Environment') { - steps { - echo 'Preparing the environment' - sh 'docker image prune --all' - sh 'docker login --username "${DOCKER_HUB_USR}" --password "${DOCKER_HUB_PSW}"' - } + agent { + label 'docker' } - - stage('Build Ubuntu GNU') { - steps { - echo "Building ${env.UBUNTU_GNU_IMAGE_TAG}" - sh 'docker build --tag "${UBUNTU_GNU_IMAGE_TAG}" --file "${WORKSPACE}/Docker/Dockerfile.${UBUNTU_GNU_IMAGE_NAME}" "${WORKSPACE}"' - } + + environment { + DOCKER_HUB = credentials('DockerHubNOAAEPIC') + UBUNTU_GNU_IMAGE_NAME = 'ubuntu20.04-gnu9.3' + UBUNTU_GNU_IMAGE_VERSION = '0.1' + UBUNTU_GNU_IMAGE_TAG = "${env.DOCKER_HUB_USR}/${env.UBUNTU_GNU_IMAGE_NAME}:${env.UBUNTU_GNU_IMAGE_VERSION}" + UBUNTU_GNU_HPCSTACK_IMAGE_NAME = "${env.UBUNTU_GNU_IMAGE_NAME}-hpc-stack" + UBUNTU_GNU_HPCSTACK_IMAGE_VERSION = '0.1' + UBUNTU_GNU_HPCSTACK_IMAGE_TAG = "${env.DOCKER_HUB_USR}/${env.UBUNTU_GNU_HPCSTACK_IMAGE_NAME}:${env.UBUNTU_GNU_HPCSTACK_IMAGE_VERSION}" + UBUNTU_GNU_SRW_IMAGE_NAME = "${env.UBUNTU_GNU_IMAGE_NAME}-ufs-srwapp" + UBUNTU_GNU_SRW_IMAGE_VERSION = '0.1' + UBUNTU_GNU_SRW_IMAGE_TAG = "${env.DOCKER_HUB_USR}/${env.UBUNTU_GNU_SRW_IMAGE_NAME}:${env.UBUNTU_GNU_SRW_IMAGE_VERSION}" + UBUNTU_GNU_DATA_IMAGE_NAME = "input-data" + UBUNTU_GNU_DATA_IMAGE_VERSION = '20220414' + UBUNTU_GNU_DATA_IMAGE_TAG = "${env.DOCKER_HUB_USR}/${env.UBUNTU_GNU_DATA_IMAGE_NAME}:${env.UBUNTU_GNU_DATA_IMAGE_VERSION}" + } - - stage('Test Ubuntu GNU') { - steps { - echo "Testing ${env.UBUNTU_GNU_IMAGE_TAG}" - sh 'docker run "${UBUNTU_GNU_IMAGE_TAG}" gfortran --version' + + stages { + stage('Prepare Environment') { + steps { + echo 'Preparing the environment' + sh 'docker image prune --all' + sh 'docker login --username "${DOCKER_HUB_USR}" --password "${DOCKER_HUB_PSW}"' + } } - } - - stage('Release Ubuntu GNU') { - when { - branch 'main' + + stage('Build Ubuntu GNU') { + steps { + echo "Building ${env.UBUNTU_GNU_IMAGE_TAG}" + sh 'docker build --tag "${UBUNTU_GNU_IMAGE_TAG}" --file "${WORKSPACE}/Docker/Dockerfile.${UBUNTU_GNU_IMAGE_NAME}" "${WORKSPACE}"' + } } - - steps { - echo "Releasing ${env.UBUNTU_GNU_IMAGE_TAG}" - sh 'docker push "${UBUNTU_GNU_IMAGE_TAG}"' + + stage('Test Ubuntu GNU') { + steps { + echo "Testing ${env.UBUNTU_GNU_IMAGE_TAG}" + sh 'docker run "${UBUNTU_GNU_IMAGE_TAG}" gfortran --version' + } } - } - - stage('Build Ubuntu GNU HPC Stack') { - steps { - echo "Building ${env.UBUNTU_GNU_HPCSTACK_IMAGE_TAG}" - sh 'docker build --tag "${UBUNTU_GNU_HPCSTACK_IMAGE_TAG}" --file "${WORKSPACE}/Docker/Dockerfile.${UBUNTU_GNU_HPCSTACK_IMAGE_NAME}" "${WORKSPACE}"' + + stage('Release Ubuntu GNU') { + when { + branch 'main' + } + + steps { + echo "Releasing ${env.UBUNTU_GNU_IMAGE_TAG}" + sh 'docker push "${UBUNTU_GNU_IMAGE_TAG}"' + } + } + + stage('Build Ubuntu GNU HPC Stack') { + steps { + echo "Building ${env.UBUNTU_GNU_HPCSTACK_IMAGE_TAG}" + sh 'docker build --tag "${UBUNTU_GNU_HPCSTACK_IMAGE_TAG}" --file "${WORKSPACE}/Docker/Dockerfile.${UBUNTU_GNU_HPCSTACK_IMAGE_NAME}" "${WORKSPACE}"' + } + } + + stage('Test Ubuntu GNU HPC Stack') { + steps { + echo "Testing ${env.UBUNTU_GNU_HPCSTACK_IMAGE_TAG}" + sh 'docker run "${UBUNTU_GNU_HPCSTACK_IMAGE_TAG}"' + } + } + + stage('Release Ubuntu GNU HPC Stack') { + when { + branch 'main' + } + + steps { + echo "Releasing ${env.UBUNTU_GNU_HPCSTACK_IMAGE_TAG}" + sh 'docker push "${UBUNTU_GNU_HPCSTACK_IMAGE_TAG}"' + } + } + + stage('Build Ubuntu GNU SRW') { + steps { + echo "Building ${env.UBUNTU_GNU_SRW_IMAGE_TAG}" + sh 'docker build --tag "${UBUNTU_GNU_SRW_IMAGE_TAG}" --file "${WORKSPACE}/Docker/Dockerfile.${UBUNTU_GNU_SRW_IMAGE_NAME}" "${WORKSPACE}"' } } - stage('Test Ubuntu GNU HPC Stack') { + stage('Test Ubuntu GNU SRW') { steps { - echo "Testing ${env.UBUNTU_GNU_HPCSTACK_IMAGE_TAG}" - sh 'docker run "${UBUNTU_GNU_HPCSTACK_IMAGE_TAG}"' + echo "Testing ${env.UBUNTU_GNU_SRW_IMAGE_TAG}" + sh 'docker run "${UBUNTU_GNU_SRW_IMAGE_TAG}"' } } - stage('Release Ubuntu GNU HPC Stack') { + stage('Release Ubuntu GNU SRW') { when { branch 'main' } steps { - echo "Releasing ${env.UBUNTU_GNU_HPCSTACK_IMAGE_TAG}" - sh 'docker push "${UBUNTU_GNU_HPCSTACK_IMAGE_TAG}"' + echo "Releasing ${env.UBUNTU_GNU_SRW_IMAGE_TAG}" + sh 'docker push "${UBUNTU_GNU_SRW_IMAGE_TAG}"' } } - stage('Build Ubuntu GNU SRW') { - steps { - echo "Building ${env.UBUNTU_GNU_SRW_IMAGE_TAG}" - sh 'docker build --tag "${UBUNTU_GNU_SRW_IMAGE_TAG}" --file "${WORKSPACE}/Docker/Dockerfile.${UBUNTU_GNU_SRW_IMAGE_NAME}" "${WORKSPACE}"' + + stage('RUN DATA SCRIPT') { + steps { + echo "Pulling input-data-20220414, Creating Docker image & Pushing it to DockerHub" + sh 'PASSWORD= ${DockerHub_PASSWORD} /path/to/your/aws-s3.sh' } } - stage('Test Ubuntu GNU SRW') { + stage('Build Ubuntu GNU DATA') { + steps { + echo "Building ${env.UBUNTU_GNU_DATA_IMAGE_TAG}" + sh 'docker build --tag "${UBUNTU_GNU_DATA_IMAGE_TAG}" --file "${WORKSPACE}/Docker/Dockerfile.${UBUNTU_GNU_DATA_IMAGE_NAME}" "${WORKSPACE}"' + } + } + + stage('Test Ubuntu GNU DATA') { steps { - echo "Testing ${env.UBUNTU_GNU_SRW_IMAGE_TAG}" - sh 'docker run "${UBUNTU_GNU_SRW_IMAGE_TAG}"' + echo "Testing ${env.UBUNTU_GNU_DATA_IMAGE_TAG}" + sh 'docker run "${UBUNTU_GNU_DATA_IMAGE_TAG}"' } } - stage('Release Ubuntu GNU SRW') { + stage('Release Ubuntu GNU DATA') { when { branch 'main' } steps { - echo "Releasing ${env.UBUNTU_GNU_SRW_IMAGE_TAG}" - sh 'docker push "${UBUNTU_GNU_SRW_IMAGE_TAG}"' + echo "Releasing ${env.UBUNTU_GNU_DATA_IMAGE_TAG}" + sh 'docker push "${UBUNTU_GNU_DATA_IMAGE_TAG}"' } - } + } } -} +} \ No newline at end of file From b2a8fd3b5c7c3064b5818a15f06b0667fa274432 Mon Sep 17 00:00:00 2001 From: Boris Acha <1165516@us.ray.com> Date: Fri, 19 Aug 2022 10:49:27 -0400 Subject: [PATCH 2/4] This is the shell script to download files and build image --- aws-s3.sh | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 aws-s3.sh diff --git a/aws-s3.sh b/aws-s3.sh new file mode 100644 index 0000000..4bf8474 --- /dev/null +++ b/aws-s3.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +declare -a subfolders=("FV3_input_data_RRTMGP" "FV3_input_data_gsd" "FV3_input_frac" "FV3_regional_input_data" "GOCART" "MOM6_FIX" "MOM6_IC" "fv3_regional_control" ) + +PASSWORD=${PASSWORD:-'abc'} + +cd /to/path/ +mkdir input-data-20220414 +cd input-data-20220414 + +for i in "${subfolders[@]}"; do + + mkdir ${i} + cd ${i} + aws s3 cp --no-sign-request s3://noaa-ufs-regtests-pds/input-data-20220414/${i}/ . --recursive + cd .. + +done From a54629751d9c8209f885cfee93eec266eb17024e Mon Sep 17 00:00:00 2001 From: Boris Acha <1165516@us.ray.com> Date: Fri, 19 Aug 2022 11:02:29 -0400 Subject: [PATCH 3/4] Updated Script was missing part --- aws-s3.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/aws-s3.sh b/aws-s3.sh index 4bf8474..1e6fe89 100644 --- a/aws-s3.sh +++ b/aws-s3.sh @@ -16,3 +16,14 @@ for i in "${subfolders[@]}"; do cd .. done + +## Build, Tag & Push Data Container To DockerHub + +cd /path/to/s3/test-script/ +git clone https://github.com/jkbk2004/fv3-input-data +cp -r input-data-20220414 fv3-input-data +cd fv3-input-data +docker build -t my_image . +docker tag my_image noaaepic/input-data:20220414 +docker login --username noaaepic --password=$PASSWORD +docker push noaaepic/input-data:20220414 \ No newline at end of file From b186bb63edd6c71534a26a5611880a92171d45d4 Mon Sep 17 00:00:00 2001 From: Boris Acha <1165516@us.ray.com> Date: Thu, 25 Aug 2022 14:26:50 -0400 Subject: [PATCH 4/4] Updated shell script for data container --- aws-s3.sh | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/aws-s3.sh b/aws-s3.sh index 1e6fe89..659616c 100644 --- a/aws-s3.sh +++ b/aws-s3.sh @@ -1,29 +1,31 @@ #!/bin/bash - -declare -a subfolders=("FV3_input_data_RRTMGP" "FV3_input_data_gsd" "FV3_input_frac" "FV3_regional_input_data" "GOCART" "MOM6_FIX" "MOM6_IC" "fv3_regional_control" ) - +declare -a subfolders=("CICE_FIX/100" "CICE_IC/100" "CPL_FIX/aC96o100" "FV3_fix/fix_co2_proj" "FV3_fix_tiled/C96" "FV3_input_data/INPUT" "FV3_input_data/INPUT_L127" "FV3_input_data/INPUT_L127_mx100" "FV3_input_data/ORO_FLAKE" "FV3_input_data/RESTART" "FV3_input_data_gsd/FV3_input_data_C96_with_aerosols" "FV3_input_data_gsd/drag_suite" "FV3_input_data_INCCN_aeroclim/mg2_IN_CCN" "FV3_input_data_INCCN_aeroclim/MERRA2" "FV3_input_data_INCCN_aeroclim/aer_data" "FV3_input_data_INCCN_aeroclim/aer_data/LUTS" "FV3_input_frac/C96_L64.mx100_frac" "FV3_input_frac/C96_L127.mx100_frac" "MOM6_IC/100" "MOM6_IC/100/2011100100" "MOM6_FIX/100" "fv3_regional_control/INPUT" "fv3_regional_control/RESTART" "GOCART/p7/ExtData/dust" "GOCART/p7/ExtData/MERRA2/sfc" "GOCART/p7/ExtData/QFED/2013/04" "GOCART/p7/ExtData/QFED/2013/save03" "GOCART/p7/ExtData/CEDS/v2019/2013" "GOCART/p7/ExtData/monochromatic" "GOCART/p7/ExtData/optics" "GOCART/p7/ExtData/MEGAN_OFFLINE_BVOC/v2019-10/2013" "GOCART/p7/ExtData/PIESA/L127" "GOCART/p7/ExtData/PIESA/sfc/HTAP/v2.2" "GOCART/p7/ExtData/volcanic" "GOCART/p7/rc" "GOCART/p8/ExtData/dust" "GOCART/p8/ExtData/MERRA2/sfc" "GOCART/p8/ExtData/QFED/2021/03" "GOCART/p8/ExtData/QFED/2013/04" "GOCART/p8/ExtData/CEDS/v2019/2021" "GOCART/p8/ExtData/CEDS/v2019/2013" "GOCART/p8/ExtData/monochromatic" "GOCART/p8/ExtData/optics" "GOCART/p8/ExtData/MEGAN_OFFLINE_BVOC/v2019-10/2021" "GOCART/p8/ExtData/MEGAN_OFFLINE_BVOC/v2019-10/2013" "GOCART/p8/ExtData/PIESA/L127" "GOCART/p8/ExtData/PIESA/sfc/HTAP/v2.2" "GOCART/p8/ExtData/volcanic" "GOCART/p8/rc" "GOCART/ExtData/dust" "GOCART/ExtData/monochromatic" "GOCART/ExtData/optics" "GOCART/ExtData/PIESA/sfc/HTAP/v2.2" "GOCART/ExtData/PIESA/sfc/QFED/NRT/v2.5" "GOCART/ExtData/volcanic" "GOCART/rc") PASSWORD=${PASSWORD:-'abc'} -cd /to/path/ +#update line number 6, 13, and 15 with path where script will be running + +cd /home/ubuntu/data-container mkdir input-data-20220414 cd input-data-20220414 - for i in "${subfolders[@]}"; do - - mkdir ${i} - cd ${i} - aws s3 cp --no-sign-request s3://noaa-ufs-regtests-pds/input-data-20220414/${i}/ . --recursive - cd .. + mkdir -p ${i} + cd /home/ubuntu/data-container/input-data-20220414/${i} + aws s3 cp --no-sign-request s3://noaa-ufs-regtests-pds/input-data-20220414/${i}/ . --recursive + cd /home/ubuntu/data-container/input-data-20220414/ done + + ## Build, Tag & Push Data Container To DockerHub + +##update line 24 with path to the directory with the directory where input-data-20220414 lies cd /path/to/s3/test-script/ -git clone https://github.com/jkbk2004/fv3-input-data -cp -r input-data-20220414 fv3-input-data -cd fv3-input-data -docker build -t my_image . -docker tag my_image noaaepic/input-data:20220414 -docker login --username noaaepic --password=$PASSWORD -docker push noaaepic/input-data:20220414 \ No newline at end of file +sudo git clone https://github.com/jkbk2004/fv3-input-data +sudo cp -r input-data-20220414 fv3-input-data +sudo cd fv3-input-data +sudo docker login --username noaaepic --password=$PASSWORD +sudo docker build -t my_image . +sudo docker tag my_image noaaepic/input-data:20220414 +sudo docker push noaaepic/input-data:20220414 \ No newline at end of file