diff --git a/Jenkinsfile b/Jenkinsfile index 25c948f..e746da7 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,103 +1,140 @@ pipeline { - agent { - label 'docker' - } - - environment { - DOCKER_HUB = credentials('DockerHubNOAAEPIC') - UBUNTU_GNU_IMAGE_NAME = 'ubuntu20.04-gnu9.3' - UBUNTU_GNU_IMAGE_VERSION = '0.1' - UBUNTU_GNU_IMAGE_TAG = "${env.DOCKER_HUB_USR}/${env.UBUNTU_GNU_IMAGE_NAME}:${env.UBUNTU_GNU_IMAGE_VERSION}" - UBUNTU_GNU_HPCSTACK_IMAGE_NAME = "${env.UBUNTU_GNU_IMAGE_NAME}-hpc-stack" - UBUNTU_GNU_HPCSTACK_IMAGE_VERSION = '0.1' - UBUNTU_GNU_HPCSTACK_IMAGE_TAG = "${env.DOCKER_HUB_USR}/${env.UBUNTU_GNU_HPCSTACK_IMAGE_NAME}:${env.UBUNTU_GNU_HPCSTACK_IMAGE_VERSION}" - UBUNTU_GNU_SRW_IMAGE_NAME = "${env.UBUNTU_GNU_IMAGE_NAME}-ufs-srwapp" - UBUNTU_GNU_SRW_IMAGE_VERSION = '0.1' - UBUNTU_GNU_SRW_IMAGE_TAG = "${env.DOCKER_HUB_USR}/${env.UBUNTU_GNU_SRW_IMAGE_NAME}:${env.UBUNTU_GNU_SRW_IMAGE_VERSION}" - } - - stages { - stage('Prepare Environment') { - steps { - echo 'Preparing the environment' - sh 'docker image prune --all' - sh 'docker login --username "${DOCKER_HUB_USR}" --password "${DOCKER_HUB_PSW}"' - } + agent { + label 'docker' } - - stage('Build Ubuntu GNU') { - steps { - echo "Building ${env.UBUNTU_GNU_IMAGE_TAG}" - sh 'docker build --tag "${UBUNTU_GNU_IMAGE_TAG}" --file "${WORKSPACE}/Docker/Dockerfile.${UBUNTU_GNU_IMAGE_NAME}" "${WORKSPACE}"' - } + + environment { + DOCKER_HUB = credentials('DockerHubNOAAEPIC') + UBUNTU_GNU_IMAGE_NAME = 'ubuntu20.04-gnu9.3' + UBUNTU_GNU_IMAGE_VERSION = '0.1' + UBUNTU_GNU_IMAGE_TAG = "${env.DOCKER_HUB_USR}/${env.UBUNTU_GNU_IMAGE_NAME}:${env.UBUNTU_GNU_IMAGE_VERSION}" + UBUNTU_GNU_HPCSTACK_IMAGE_NAME = "${env.UBUNTU_GNU_IMAGE_NAME}-hpc-stack" + UBUNTU_GNU_HPCSTACK_IMAGE_VERSION = '0.1' + UBUNTU_GNU_HPCSTACK_IMAGE_TAG = "${env.DOCKER_HUB_USR}/${env.UBUNTU_GNU_HPCSTACK_IMAGE_NAME}:${env.UBUNTU_GNU_HPCSTACK_IMAGE_VERSION}" + UBUNTU_GNU_SRW_IMAGE_NAME = "${env.UBUNTU_GNU_IMAGE_NAME}-ufs-srwapp" + UBUNTU_GNU_SRW_IMAGE_VERSION = '0.1' + UBUNTU_GNU_SRW_IMAGE_TAG = "${env.DOCKER_HUB_USR}/${env.UBUNTU_GNU_SRW_IMAGE_NAME}:${env.UBUNTU_GNU_SRW_IMAGE_VERSION}" + UBUNTU_GNU_DATA_IMAGE_NAME = "input-data" + UBUNTU_GNU_DATA_IMAGE_VERSION = '20220414' + UBUNTU_GNU_DATA_IMAGE_TAG = "${env.DOCKER_HUB_USR}/${env.UBUNTU_GNU_DATA_IMAGE_NAME}:${env.UBUNTU_GNU_DATA_IMAGE_VERSION}" + } - - stage('Test Ubuntu GNU') { - steps { - echo "Testing ${env.UBUNTU_GNU_IMAGE_TAG}" - sh 'docker run "${UBUNTU_GNU_IMAGE_TAG}" gfortran --version' + + stages { + stage('Prepare Environment') { + steps { + echo 'Preparing the environment' + sh 'docker image prune --all' + sh 'docker login --username "${DOCKER_HUB_USR}" --password "${DOCKER_HUB_PSW}"' + } } - } - - stage('Release Ubuntu GNU') { - when { - branch 'main' + + stage('Build Ubuntu GNU') { + steps { + echo "Building ${env.UBUNTU_GNU_IMAGE_TAG}" + sh 'docker build --tag "${UBUNTU_GNU_IMAGE_TAG}" --file "${WORKSPACE}/Docker/Dockerfile.${UBUNTU_GNU_IMAGE_NAME}" "${WORKSPACE}"' + } } - - steps { - echo "Releasing ${env.UBUNTU_GNU_IMAGE_TAG}" - sh 'docker push "${UBUNTU_GNU_IMAGE_TAG}"' + + stage('Test Ubuntu GNU') { + steps { + echo "Testing ${env.UBUNTU_GNU_IMAGE_TAG}" + sh 'docker run "${UBUNTU_GNU_IMAGE_TAG}" gfortran --version' + } } - } - - stage('Build Ubuntu GNU HPC Stack') { - steps { - echo "Building ${env.UBUNTU_GNU_HPCSTACK_IMAGE_TAG}" - sh 'docker build --tag "${UBUNTU_GNU_HPCSTACK_IMAGE_TAG}" --file "${WORKSPACE}/Docker/Dockerfile.${UBUNTU_GNU_HPCSTACK_IMAGE_NAME}" "${WORKSPACE}"' + + stage('Release Ubuntu GNU') { + when { + branch 'main' + } + + steps { + echo "Releasing ${env.UBUNTU_GNU_IMAGE_TAG}" + sh 'docker push "${UBUNTU_GNU_IMAGE_TAG}"' + } + } + + stage('Build Ubuntu GNU HPC Stack') { + steps { + echo "Building ${env.UBUNTU_GNU_HPCSTACK_IMAGE_TAG}" + sh 'docker build --tag "${UBUNTU_GNU_HPCSTACK_IMAGE_TAG}" --file "${WORKSPACE}/Docker/Dockerfile.${UBUNTU_GNU_HPCSTACK_IMAGE_NAME}" "${WORKSPACE}"' + } + } + + stage('Test Ubuntu GNU HPC Stack') { + steps { + echo "Testing ${env.UBUNTU_GNU_HPCSTACK_IMAGE_TAG}" + sh 'docker run "${UBUNTU_GNU_HPCSTACK_IMAGE_TAG}"' + } + } + + stage('Release Ubuntu GNU HPC Stack') { + when { + branch 'main' + } + + steps { + echo "Releasing ${env.UBUNTU_GNU_HPCSTACK_IMAGE_TAG}" + sh 'docker push "${UBUNTU_GNU_HPCSTACK_IMAGE_TAG}"' + } + } + + stage('Build Ubuntu GNU SRW') { + steps { + echo "Building ${env.UBUNTU_GNU_SRW_IMAGE_TAG}" + sh 'docker build --tag "${UBUNTU_GNU_SRW_IMAGE_TAG}" --file "${WORKSPACE}/Docker/Dockerfile.${UBUNTU_GNU_SRW_IMAGE_NAME}" "${WORKSPACE}"' } } - stage('Test Ubuntu GNU HPC Stack') { + stage('Test Ubuntu GNU SRW') { steps { - echo "Testing ${env.UBUNTU_GNU_HPCSTACK_IMAGE_TAG}" - sh 'docker run "${UBUNTU_GNU_HPCSTACK_IMAGE_TAG}"' + echo "Testing ${env.UBUNTU_GNU_SRW_IMAGE_TAG}" + sh 'docker run "${UBUNTU_GNU_SRW_IMAGE_TAG}"' } } - stage('Release Ubuntu GNU HPC Stack') { + stage('Release Ubuntu GNU SRW') { when { branch 'main' } steps { - echo "Releasing ${env.UBUNTU_GNU_HPCSTACK_IMAGE_TAG}" - sh 'docker push "${UBUNTU_GNU_HPCSTACK_IMAGE_TAG}"' + echo "Releasing ${env.UBUNTU_GNU_SRW_IMAGE_TAG}" + sh 'docker push "${UBUNTU_GNU_SRW_IMAGE_TAG}"' } } - stage('Build Ubuntu GNU SRW') { - steps { - echo "Building ${env.UBUNTU_GNU_SRW_IMAGE_TAG}" - sh 'docker build --tag "${UBUNTU_GNU_SRW_IMAGE_TAG}" --file "${WORKSPACE}/Docker/Dockerfile.${UBUNTU_GNU_SRW_IMAGE_NAME}" "${WORKSPACE}"' + + stage('RUN DATA SCRIPT') { + steps { + echo "Pulling input-data-20220414, Creating Docker image & Pushing it to DockerHub" + sh 'PASSWORD= ${DockerHub_PASSWORD} /path/to/your/aws-s3.sh' } } - stage('Test Ubuntu GNU SRW') { + stage('Build Ubuntu GNU DATA') { + steps { + echo "Building ${env.UBUNTU_GNU_DATA_IMAGE_TAG}" + sh 'docker build --tag "${UBUNTU_GNU_DATA_IMAGE_TAG}" --file "${WORKSPACE}/Docker/Dockerfile.${UBUNTU_GNU_DATA_IMAGE_NAME}" "${WORKSPACE}"' + } + } + + stage('Test Ubuntu GNU DATA') { steps { - echo "Testing ${env.UBUNTU_GNU_SRW_IMAGE_TAG}" - sh 'docker run "${UBUNTU_GNU_SRW_IMAGE_TAG}"' + echo "Testing ${env.UBUNTU_GNU_DATA_IMAGE_TAG}" + sh 'docker run "${UBUNTU_GNU_DATA_IMAGE_TAG}"' } } - stage('Release Ubuntu GNU SRW') { + stage('Release Ubuntu GNU DATA') { when { branch 'main' } steps { - echo "Releasing ${env.UBUNTU_GNU_SRW_IMAGE_TAG}" - sh 'docker push "${UBUNTU_GNU_SRW_IMAGE_TAG}"' + echo "Releasing ${env.UBUNTU_GNU_DATA_IMAGE_TAG}" + sh 'docker push "${UBUNTU_GNU_DATA_IMAGE_TAG}"' } - } + } } -} +} \ No newline at end of file diff --git a/aws-s3.sh b/aws-s3.sh new file mode 100644 index 0000000..659616c --- /dev/null +++ b/aws-s3.sh @@ -0,0 +1,31 @@ +#!/bin/bash +declare -a subfolders=("CICE_FIX/100" "CICE_IC/100" "CPL_FIX/aC96o100" "FV3_fix/fix_co2_proj" "FV3_fix_tiled/C96" "FV3_input_data/INPUT" "FV3_input_data/INPUT_L127" "FV3_input_data/INPUT_L127_mx100" "FV3_input_data/ORO_FLAKE" "FV3_input_data/RESTART" "FV3_input_data_gsd/FV3_input_data_C96_with_aerosols" "FV3_input_data_gsd/drag_suite" "FV3_input_data_INCCN_aeroclim/mg2_IN_CCN" "FV3_input_data_INCCN_aeroclim/MERRA2" "FV3_input_data_INCCN_aeroclim/aer_data" "FV3_input_data_INCCN_aeroclim/aer_data/LUTS" "FV3_input_frac/C96_L64.mx100_frac" "FV3_input_frac/C96_L127.mx100_frac" "MOM6_IC/100" "MOM6_IC/100/2011100100" "MOM6_FIX/100" "fv3_regional_control/INPUT" "fv3_regional_control/RESTART" "GOCART/p7/ExtData/dust" "GOCART/p7/ExtData/MERRA2/sfc" "GOCART/p7/ExtData/QFED/2013/04" "GOCART/p7/ExtData/QFED/2013/save03" "GOCART/p7/ExtData/CEDS/v2019/2013" "GOCART/p7/ExtData/monochromatic" "GOCART/p7/ExtData/optics" "GOCART/p7/ExtData/MEGAN_OFFLINE_BVOC/v2019-10/2013" "GOCART/p7/ExtData/PIESA/L127" "GOCART/p7/ExtData/PIESA/sfc/HTAP/v2.2" "GOCART/p7/ExtData/volcanic" "GOCART/p7/rc" "GOCART/p8/ExtData/dust" "GOCART/p8/ExtData/MERRA2/sfc" "GOCART/p8/ExtData/QFED/2021/03" "GOCART/p8/ExtData/QFED/2013/04" "GOCART/p8/ExtData/CEDS/v2019/2021" "GOCART/p8/ExtData/CEDS/v2019/2013" "GOCART/p8/ExtData/monochromatic" "GOCART/p8/ExtData/optics" "GOCART/p8/ExtData/MEGAN_OFFLINE_BVOC/v2019-10/2021" "GOCART/p8/ExtData/MEGAN_OFFLINE_BVOC/v2019-10/2013" "GOCART/p8/ExtData/PIESA/L127" "GOCART/p8/ExtData/PIESA/sfc/HTAP/v2.2" "GOCART/p8/ExtData/volcanic" "GOCART/p8/rc" "GOCART/ExtData/dust" "GOCART/ExtData/monochromatic" "GOCART/ExtData/optics" "GOCART/ExtData/PIESA/sfc/HTAP/v2.2" "GOCART/ExtData/PIESA/sfc/QFED/NRT/v2.5" "GOCART/ExtData/volcanic" "GOCART/rc") +PASSWORD=${PASSWORD:-'abc'} + +#update line number 6, 13, and 15 with path where script will be running + +cd /home/ubuntu/data-container +mkdir input-data-20220414 +cd input-data-20220414 +for i in "${subfolders[@]}"; do + + mkdir -p ${i} + cd /home/ubuntu/data-container/input-data-20220414/${i} + aws s3 cp --no-sign-request s3://noaa-ufs-regtests-pds/input-data-20220414/${i}/ . --recursive + cd /home/ubuntu/data-container/input-data-20220414/ +done + + + +## Build, Tag & Push Data Container To DockerHub + + +##update line 24 with path to the directory with the directory where input-data-20220414 lies +cd /path/to/s3/test-script/ +sudo git clone https://github.com/jkbk2004/fv3-input-data +sudo cp -r input-data-20220414 fv3-input-data +sudo cd fv3-input-data +sudo docker login --username noaaepic --password=$PASSWORD +sudo docker build -t my_image . +sudo docker tag my_image noaaepic/input-data:20220414 +sudo docker push noaaepic/input-data:20220414 \ No newline at end of file