diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile index 6a2e064be0..8b987a16fa 100644 --- a/ci/Jenkinsfile +++ b/ci/Jenkinsfile @@ -221,7 +221,11 @@ pipeline { def yaml_case = readYaml file: "${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp" def build_system = yaml_case.experiment.system try { + // Remove any data in that might still be in the DATAROOT directory from previous runs + STMP=sh(script: "${HOMEgfs}/ci/scripts/utils/get_config_var.py STMP ${CUSTOM_WORKSPACE}/RUNTESTS/EXPDIR/${pslot}", returnStdout: true).trim() + sh(script: "rm -Rf ${STMP}/RUNDIRS/${pslot}") sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} ${build_system}") + sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cleanup_experiment ${CUSTOM_WORKSPACE}/RUNTESTS/EXPDIR/${pslot}") } catch (Exception error_experment) { sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_batch_jobs ${pslot}") ws(CUSTOM_WORKSPACE) { diff --git a/ci/scripts/check_ci.sh b/ci/scripts/check_ci.sh index 825d8f5e8b..a89a661042 100755 --- a/ci/scripts/check_ci.sh +++ b/ci/scripts/check_ci.sh @@ -168,8 +168,7 @@ for pr in ${pr_list}; do fi if [[ "${rocoto_state}" == "DONE" ]]; then #Remove Experment cases that completed successfully - rm -Rf "${pslot_dir}" - rm -Rf "${pr_dir}/RUNTESTS/COMROOT/${pslot}" + "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh" cleanup_experiment "${pslot_dir}" rm -f "${output_ci_single}" # echo "\`\`\`" > "${output_ci_single}" DATE=$(date +'%D %r') diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh index 2a51467d38..64988986fe 100755 --- a/ci/scripts/utils/ci_utils.sh +++ b/ci/scripts/utils/ci_utils.sh @@ -137,7 +137,6 @@ function publish_logs() { local PR_header="$1" local dir_path="$2" local file="$3" - local full_paths="" while IFS= read -r line; do full_path="${dir_path}/${line}" @@ -155,3 +154,24 @@ function publish_logs() { fi echo "${URL}" } + +function cleanup_experiment() { + + local EXPDIR="$1" + local pslot + local ARCDIR + local ATARDIR + local COMROOT + + EXPDIR="$1" + pslot=$(basename "${EXPDIR}") + + # Use the Python utility to get the required variables + read -r ARCDIR ATARDIR STMP COMROOT < <("${HOMEgfs}/ci/scripts/utils/get_config_var.py" ARCDIR ATARDIR STMP COMROOT "${EXPDIR}") || true + + rm -Rf "${STMP}/RUNDIRS/${pslot}" + rm -Rf "${ARCDIR:?}" + rm -Rf "${ATARDIR:?}" + rm -Rf "${COMROOT:?}" + rm -Rf "${EXPDIR}" +} diff --git a/ci/scripts/utils/get_config_var.py b/ci/scripts/utils/get_config_var.py new file mode 100755 index 0000000000..3e63fcf376 --- /dev/null +++ b/ci/scripts/utils/get_config_var.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 + +import os +import argparse +from wxflow import Configuration + + +def get_config_vars(var_names, config_path): + """ + GET_CONFIG_VARS Get configuration variables from a config file or directory. + Parameters: + var_names (list of str): The names of the configuration variables to retrieve. + config_path (str): The path to the configuration file or directory. + Returns: + list of str: The values of the specified configuration variables. + """ + if os.path.isfile(config_path): + config_dir = os.path.dirname(config_path) + config_file = os.path.basename(config_path) + elif os.path.isdir(config_path): + config_dir = config_path + config_file = 'config.base' + config = Configuration(config_dir) + config_data = config.parse_config(config_file) + return [config_data[var_name] for var_name in var_names] + + +if __name__ == "__main__": + """ + Main entry point for the script. + Parses command-line arguments and retrieves the specified configuration variables. + """ + parser = argparse.ArgumentParser(description="Get configuration variables from a config file or directory.") + parser.add_argument("var_names", nargs='+', help="The names of the configuration variables to retrieve.") + parser.add_argument("config_path", help="The path to the configuration file or directory.") + + args = parser.parse_args() + + var_names = args.var_names + config_path = args.config_path + + values = get_config_vars(var_names, config_path) + print(" ".join(values))