New data container jenkinsfile #9
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,103 +1,140 @@ | ||
| pipeline { | ||
| agent { | ||
| label 'docker' | ||
| } | ||
|
|
||
| environment { | ||
| DOCKER_HUB = credentials('DockerHubNOAAEPIC') | ||
| UBUNTU_GNU_IMAGE_NAME = 'ubuntu20.04-gnu9.3' | ||
| UBUNTU_GNU_IMAGE_VERSION = '0.1' | ||
| UBUNTU_GNU_IMAGE_TAG = "${env.DOCKER_HUB_USR}/${env.UBUNTU_GNU_IMAGE_NAME}:${env.UBUNTU_GNU_IMAGE_VERSION}" | ||
| UBUNTU_GNU_HPCSTACK_IMAGE_NAME = "${env.UBUNTU_GNU_IMAGE_NAME}-hpc-stack" | ||
| UBUNTU_GNU_HPCSTACK_IMAGE_VERSION = '0.1' | ||
| UBUNTU_GNU_HPCSTACK_IMAGE_TAG = "${env.DOCKER_HUB_USR}/${env.UBUNTU_GNU_HPCSTACK_IMAGE_NAME}:${env.UBUNTU_GNU_HPCSTACK_IMAGE_VERSION}" | ||
| UBUNTU_GNU_SRW_IMAGE_NAME = "${env.UBUNTU_GNU_IMAGE_NAME}-ufs-srwapp" | ||
| UBUNTU_GNU_SRW_IMAGE_VERSION = '0.1' | ||
| UBUNTU_GNU_SRW_IMAGE_TAG = "${env.DOCKER_HUB_USR}/${env.UBUNTU_GNU_SRW_IMAGE_NAME}:${env.UBUNTU_GNU_SRW_IMAGE_VERSION}" | ||
| UBUNTU_GNU_DATA_IMAGE_NAME = "input-data" | ||
| UBUNTU_GNU_DATA_IMAGE_VERSION = '20220414' | ||
| UBUNTU_GNU_DATA_IMAGE_TAG = "${env.DOCKER_HUB_USR}/${env.UBUNTU_GNU_DATA_IMAGE_NAME}:${env.UBUNTU_GNU_DATA_IMAGE_VERSION}" | ||
|
|
||
| } | ||
|
|
||
| stages { | ||
| stage('Prepare Environment') { | ||
| steps { | ||
| echo 'Preparing the environment' | ||
| sh 'docker image prune --all' | ||
| sh 'docker login --username "${DOCKER_HUB_USR}" --password "${DOCKER_HUB_PSW}"' | ||
| } | ||
| } | ||
|
|
||
| stage('Build Ubuntu GNU') { | ||
| steps { | ||
| echo "Building ${env.UBUNTU_GNU_IMAGE_TAG}" | ||
| sh 'docker build --tag "${UBUNTU_GNU_IMAGE_TAG}" --file "${WORKSPACE}/Docker/Dockerfile.${UBUNTU_GNU_IMAGE_NAME}" "${WORKSPACE}"' | ||
| } | ||
| } | ||
|
|
||
| stage('Test Ubuntu GNU') { | ||
| steps { | ||
| echo "Testing ${env.UBUNTU_GNU_IMAGE_TAG}" | ||
| sh 'docker run "${UBUNTU_GNU_IMAGE_TAG}" gfortran --version' | ||
| } | ||
| } | ||
|
|
||
| stage('Release Ubuntu GNU') { | ||
| when { | ||
| branch 'main' | ||
| } | ||
|
|
||
| steps { | ||
| echo "Releasing ${env.UBUNTU_GNU_IMAGE_TAG}" | ||
| sh 'docker push "${UBUNTU_GNU_IMAGE_TAG}"' | ||
| } | ||
| } | ||
|
|
||
| stage('Build Ubuntu GNU HPC Stack') { | ||
| steps { | ||
| echo "Building ${env.UBUNTU_GNU_HPCSTACK_IMAGE_TAG}" | ||
| sh 'docker build --tag "${UBUNTU_GNU_HPCSTACK_IMAGE_TAG}" --file "${WORKSPACE}/Docker/Dockerfile.${UBUNTU_GNU_HPCSTACK_IMAGE_NAME}" "${WORKSPACE}"' | ||
| } | ||
| } | ||
|
|
||
| stage('Test Ubuntu GNU HPC Stack') { | ||
| steps { | ||
| echo "Testing ${env.UBUNTU_GNU_HPCSTACK_IMAGE_TAG}" | ||
| sh 'docker run "${UBUNTU_GNU_HPCSTACK_IMAGE_TAG}"' | ||
| } | ||
| } | ||
|
|
||
| stage('Release Ubuntu GNU HPC Stack') { | ||
| when { | ||
| branch 'main' | ||
| } | ||
|
|
||
| steps { | ||
| echo "Releasing ${env.UBUNTU_GNU_HPCSTACK_IMAGE_TAG}" | ||
| sh 'docker push "${UBUNTU_GNU_HPCSTACK_IMAGE_TAG}"' | ||
| } | ||
| } | ||
|
|
||
| stage('Build Ubuntu GNU SRW') { | ||
| steps { | ||
| echo "Building ${env.UBUNTU_GNU_SRW_IMAGE_TAG}" | ||
| sh 'docker build --tag "${UBUNTU_GNU_SRW_IMAGE_TAG}" --file "${WORKSPACE}/Docker/Dockerfile.${UBUNTU_GNU_SRW_IMAGE_NAME}" "${WORKSPACE}"' | ||
| } | ||
| } | ||
|
|
||
| stage('Test Ubuntu GNU SRW') { | ||
| steps { | ||
| echo "Testing ${env.UBUNTU_GNU_SRW_IMAGE_TAG}" | ||
| sh 'docker run "${UBUNTU_GNU_SRW_IMAGE_TAG}"' | ||
| } | ||
| } | ||
|
|
||
| stage('Release Ubuntu GNU SRW') { | ||
| when { | ||
| branch 'main' | ||
| } | ||
|
|
||
| steps { | ||
| echo "Releasing ${env.UBUNTU_GNU_SRW_IMAGE_TAG}" | ||
| sh 'docker push "${UBUNTU_GNU_SRW_IMAGE_TAG}"' | ||
| } | ||
| } | ||
|
|
||
|
|
||
| stage('RUN DATA SCRIPT') { | ||
| steps { | ||
| echo "Pulling input-data-20220414, Creating Docker image & Pushing it to DockerHub" | ||
| sh 'PASSWORD= ${DockerHub_PASSWORD} /path/to/your/aws-s3.sh' | ||
|
There was a problem hiding this comment. You don't need the Also, use |
||
| } | ||
| } | ||
|
|
||
| stage('Build Ubuntu GNU DATA') { | ||
| steps { | ||
| echo "Building ${env.UBUNTU_GNU_DATA_IMAGE_TAG}" | ||
| sh 'docker build --tag "${UBUNTU_GNU_DATA_IMAGE_TAG}" --file "${WORKSPACE}/Docker/Dockerfile.${UBUNTU_GNU_DATA_IMAGE_NAME}" "${WORKSPACE}"' | ||
| } | ||
| } | ||
|
|
||
| stage('Test Ubuntu GNU DATA') { | ||
| steps { | ||
| echo "Testing ${env.UBUNTU_GNU_DATA_IMAGE_TAG}" | ||
| sh 'docker run "${UBUNTU_GNU_DATA_IMAGE_TAG}"' | ||
| } | ||
| } | ||
|
|
||
| stage('Release Ubuntu GNU DATA') { | ||
| when { | ||
| branch 'main' | ||
| } | ||
|
|
||
| steps { | ||
| echo "Releasing ${env.UBUNTU_GNU_DATA_IMAGE_TAG}" | ||
| sh 'docker push "${UBUNTU_GNU_DATA_IMAGE_TAG}"' | ||
| } | ||
| } | ||
| } | ||
| } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,31 @@ | ||
| #!/bin/bash | ||
| declare -a subfolders=("CICE_FIX/100" "CICE_IC/100" "CPL_FIX/aC96o100" "FV3_fix/fix_co2_proj" "FV3_fix_tiled/C96" "FV3_input_data/INPUT" "FV3_input_data/INPUT_L127" "FV3_input_data/INPUT_L127_mx100" "FV3_input_data/ORO_FLAKE" "FV3_input_data/RESTART" "FV3_input_data_gsd/FV3_input_data_C96_with_aerosols" "FV3_input_data_gsd/drag_suite" "FV3_input_data_INCCN_aeroclim/mg2_IN_CCN" "FV3_input_data_INCCN_aeroclim/MERRA2" "FV3_input_data_INCCN_aeroclim/aer_data" "FV3_input_data_INCCN_aeroclim/aer_data/LUTS" "FV3_input_frac/C96_L64.mx100_frac" "FV3_input_frac/C96_L127.mx100_frac" "MOM6_IC/100" "MOM6_IC/100/2011100100" "MOM6_FIX/100" "fv3_regional_control/INPUT" "fv3_regional_control/RESTART" "GOCART/p7/ExtData/dust" "GOCART/p7/ExtData/MERRA2/sfc" "GOCART/p7/ExtData/QFED/2013/04" "GOCART/p7/ExtData/QFED/2013/save03" "GOCART/p7/ExtData/CEDS/v2019/2013" "GOCART/p7/ExtData/monochromatic" "GOCART/p7/ExtData/optics" "GOCART/p7/ExtData/MEGAN_OFFLINE_BVOC/v2019-10/2013" "GOCART/p7/ExtData/PIESA/L127" "GOCART/p7/ExtData/PIESA/sfc/HTAP/v2.2" "GOCART/p7/ExtData/volcanic" "GOCART/p7/rc" "GOCART/p8/ExtData/dust" "GOCART/p8/ExtData/MERRA2/sfc" "GOCART/p8/ExtData/QFED/2021/03" "GOCART/p8/ExtData/QFED/2013/04" "GOCART/p8/ExtData/CEDS/v2019/2021" "GOCART/p8/ExtData/CEDS/v2019/2013" "GOCART/p8/ExtData/monochromatic" "GOCART/p8/ExtData/optics" "GOCART/p8/ExtData/MEGAN_OFFLINE_BVOC/v2019-10/2021" "GOCART/p8/ExtData/MEGAN_OFFLINE_BVOC/v2019-10/2013" "GOCART/p8/ExtData/PIESA/L127" "GOCART/p8/ExtData/PIESA/sfc/HTAP/v2.2" "GOCART/p8/ExtData/volcanic" "GOCART/p8/rc" "GOCART/ExtData/dust" "GOCART/ExtData/monochromatic" "GOCART/ExtData/optics" "GOCART/ExtData/PIESA/sfc/HTAP/v2.2" "GOCART/ExtData/PIESA/sfc/QFED/NRT/v2.5" "GOCART/ExtData/volcanic" "GOCART/rc") | ||
| PASSWORD=${PASSWORD:-'abc'} | ||
|
|
||
| #update line number 6, 13, and 15 with path where script will be running | ||
|
There was a problem hiding this comment. You can use the variable |
||
|
|
||
| cd /home/ubuntu/data-container | ||
| mkdir input-data-20220414 | ||
| cd input-data-20220414 | ||
| for i in "${subfolders[@]}"; do | ||
|
|
||
| mkdir -p ${i} | ||
| cd /home/ubuntu/data-container/input-data-20220414/${i} | ||
| aws s3 cp --no-sign-request s3://noaa-ufs-regtests-pds/input-data-20220414/${i}/ . --recursive | ||
| cd /home/ubuntu/data-container/input-data-20220414/ | ||
| done | ||
|
|
||
|
|
||
|
|
||
| ## Build, Tag & Push Data Container To DockerHub | ||
|
|
||
|
|
||
| ##update line 24 with path to the directory with the directory where input-data-20220414 lies | ||
| cd /path/to/s3/test-script/ | ||
| sudo git clone https://github.com/jkbk2004/fv3-input-data | ||
| sudo cp -r input-data-20220414 fv3-input-data | ||
| sudo cd fv3-input-data | ||
| sudo docker login --username noaaepic --password=$PASSWORD | ||
| sudo docker build -t my_image . | ||
| sudo docker tag my_image noaaepic/input-data:20220414 | ||
| sudo docker push noaaepic/input-data:20220414 | ||
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Can you undo all of the whitespace changes? It's hard to see what's changed. I can help you on this.