Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

CI jekninsfile update hotfix #3038

Merged
merged 15 commits into from
Oct 28, 2024
Merged

CI jekninsfile update hotfix #3038

Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
b25f382
removed catchError to see error with get_host_case_list script (also …
TerryMcGuinness-NOAA Oct 26, 2024
31fa9be
catchError is still being applied, but did fail, adding source gwsetu…
TerryMcGuinness-NOAA Oct 26, 2024
688e1ba
drilling down on real error that is acctually in create experment (me…
TerryMcGuinness-NOAA Oct 26, 2024
39c644b
system is a matrix control variable and could not be resloved in shel…
TerryMcGuinness-NOAA Oct 26, 2024
9b3c6e6
removed HOME_GFS where was using system which is a matrix control var…
TerryMcGuinness-NOAA Oct 26, 2024
cf6b48e
.system not found in readYaml object and is not even used
TerryMcGuinness-NOAA Oct 26, 2024
f32df3e
drilling down to error with system that looks like write to disk problem
TerryMcGuinness-NOAA Oct 26, 2024
e115db5
fixed issue with system name clash issue
TerryMcGuinness-NOAA Oct 26, 2024
df34b7d
fixed issue with system name clash issue and put build back in for fi…
TerryMcGuinness-NOAA Oct 26, 2024
8acaa62
took out extra source on gwsetup as was not the issue
TerryMcGuinness-NOAA Oct 26, 2024
4dc4b04
failed again when running run script, wtf
TerryMcGuinness-NOAA Oct 27, 2024
24b02e9
put create experment back
TerryMcGuinness-NOAA Oct 27, 2024
ae2170a
just adding more output for finding more info on sytem fail
TerryMcGuinness-NOAA Oct 27, 2024
02c7682
finaly found bug, system was no login in matrix when added refined pa…
TerryMcGuinness-NOAA Oct 27, 2024
7c75021
put build back in to after fixing system variable bug
TerryMcGuinness-NOAA Oct 27, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 19 additions & 19 deletions ci/Jenkinsfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ pipeline {
stages {
stage('Building') {
steps {
catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') {
catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') {
script {
def HOMEgfs = "${CUSTOM_WORKSPACE}/${system}" // local HOMEgfs is used to build the system on per system basis under the custome workspace for each buile system
env.HOME_GFS = HOMEgfs // setting path in HOMEgfs as an environment variable HOME_GFS for some systems that using the path in its .bashrc
Expand Down Expand Up @@ -173,9 +173,10 @@ pipeline {
}
if (system == 'gfs') {
cases = sh(script: "${HOMEgfs}/ci/scripts/utils/get_host_case_list.py ${machine}", returnStdout: true).trim().split()
echo "Cases to run: ${cases}"
}
}
}
}
}
}
}
Expand All @@ -193,22 +194,20 @@ pipeline {
def parallelStages = cases.collectEntries { caseName ->
["${caseName}": {
stage("Create ${caseName}") {
catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') {
script {
def HOMEgfs = "${CUSTOM_WORKSPACE}/${system}" // local HOMEgfs is used to populate the XML on per system basis
env.HOME_GFS = HOMEgfs // setting path in HOMEgfs as an environment variable HOME_GFS for some systems that using the path in its .bashrc
sh(script: "sed -n '/{.*}/!p' ${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml > ${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp")
def yaml_case = readYaml file: "${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp"
system = yaml_case.experiment.system
env.RUNTESTS = "${CUSTOM_WORKSPACE}/RUNTESTS"
try {
error_output = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${caseName}.yaml", returnStdout: true).trim()
} catch (Exception error_create) {
sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "${Case} **FAILED** to create experment on ${Machine} in BUILD# ${env.BUILD_NUMBER}\n with the error:\n\\`\\`\\`\n${error_output}\\`\\`\\`" """)
error("Case ${caseName} failed to create experment directory")
}
script {
sh(script: "sed -n '/{.*}/!p' ${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml > ${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp")
def yaml_case = readYaml file: "${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp"
def build_system = yaml_case.experiment.system
def HOMEgfs = "${CUSTOM_WORKSPACE}/${build_system}" // local HOMEgfs is used to populate the XML on per system basis
env.HOME_GFS = HOMEgfs // setting path in HOMEgfs as an environment variable HOME_GFS for some systems that using the path in its .bashrc
env.RUNTESTS = "${CUSTOM_WORKSPACE}/RUNTESTS"
try {
error_output = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${caseName}.yaml", returnStdout: true).trim()
} catch (Exception error_create) {
sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "${Case} **FAILED** to create experment on ${Machine} in BUILD# ${env.BUILD_NUMBER}\n with the error:\n\\`\\`\\`\n${error_output}\\`\\`\\`" """)
error("Case ${caseName} failed to create experment directory")
}
}
}
}

stage("Running ${caseName}") {
Expand All @@ -219,8 +218,10 @@ pipeline {
def pslot = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${CUSTOM_WORKSPACE}/RUNTESTS ${caseName}", returnStdout: true).trim()
def error_file = "${CUSTOM_WORKSPACE}/RUNTESTS/${pslot}_error.logs"
sh(script: " rm -f ${error_file}")
def yaml_case = readYaml file: "${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp"
def build_system = yaml_case.experiment.system
try {
sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} ${system}")
sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} ${build_system}")
} catch (Exception error_experment) {
sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_batch_jobs ${pslot}")
ws(CUSTOM_WORKSPACE) {
Expand Down Expand Up @@ -271,7 +272,6 @@ pipeline {
}
}


stage( '5. FINALIZE' ) {
agent { label NodeName[machine].toLowerCase() }
steps {
Expand Down
Loading