From 915f2bff7c14f4625d75e5e66ae786026d09e9e7 Mon Sep 17 00:00:00 2001 From: bsaakash <11618528+bsaakash@users.noreply.github.com> Date: Mon, 29 Jul 2024 01:57:37 -0700 Subject: [PATCH 01/26] abs - formatting ucsd_uq files --- .../UCSD_UQ/calibration_utilities.py | 6 +- .../performUQ/UCSD_UQ/defaultLogLikeScript.py | 63 ++- modules/performUQ/UCSD_UQ/loglike_script.py | 13 +- modules/performUQ/UCSD_UQ/mainscript.py | 2 +- .../mainscript_hierarchical_bayesian.py | 13 +- modules/performUQ/UCSD_UQ/mainscript_tmcmc.py | 227 ++++++-- modules/performUQ/UCSD_UQ/parseData.py | 70 ++- modules/performUQ/UCSD_UQ/pdfs.py | 13 +- modules/performUQ/UCSD_UQ/processInputs.py | 54 +- modules/performUQ/UCSD_UQ/runFEM.py | 60 ++- modules/performUQ/UCSD_UQ/runTMCMC.py | 494 ++++++++++++++---- modules/performUQ/UCSD_UQ/tmcmcFunctions.py | 18 +- 12 files changed, 765 insertions(+), 268 deletions(-) diff --git a/modules/performUQ/UCSD_UQ/calibration_utilities.py b/modules/performUQ/UCSD_UQ/calibration_utilities.py index 7ddd6d4b6..5b11b053b 100644 --- a/modules/performUQ/UCSD_UQ/calibration_utilities.py +++ b/modules/performUQ/UCSD_UQ/calibration_utilities.py @@ -716,7 +716,7 @@ def _import_log_likelihood_module( os.path.join(self.workdir_main, self.log_likelihood_file_name) ) raise ImportError(msg) - return module # type: ignore + return module # type: ignore def get_log_likelihood_function(self) -> Callable: log_likelihood_module_name = os.path.splitext( @@ -740,7 +740,9 @@ def _make_mean(self, response_num: int) -> NDArray: return np.zeros((self.list_of_data_segment_lengths[response_num])) def _make_covariance(self, response_num, cov_multiplier) -> NDArray: - return cov_multiplier * np.atleast_2d(self.covariance_matrix_list[response_num]) + return cov_multiplier * np.atleast_2d( + self.covariance_matrix_list[response_num] + ) def _make_input_for_log_likelihood_function(self, prediction) -> list: return [ diff --git a/modules/performUQ/UCSD_UQ/defaultLogLikeScript.py b/modules/performUQ/UCSD_UQ/defaultLogLikeScript.py index 09c1eb6c8..37f43625c 100644 --- a/modules/performUQ/UCSD_UQ/defaultLogLikeScript.py +++ b/modules/performUQ/UCSD_UQ/defaultLogLikeScript.py @@ -12,9 +12,18 @@ def __init__(self, message): self.message = message -def log_likelihood(calibrationData, prediction, numExperiments, covarianceMatrixList, edpNamesList, edpLengthsList, - covarianceMultiplierList, scaleFactors, shiftFactors): - """ Compute the log-likelihood +def log_likelihood( + calibrationData, + prediction, + numExperiments, + covarianceMatrixList, + edpNamesList, + edpLengthsList, + covarianceMultiplierList, + scaleFactors, + shiftFactors, +): + """Compute the log-likelihood :param calibrationData: Calibration data consisting of the measured values of response. Each row contains the data from one experiment. The length of each row equals the sum of the lengths of all response quantities. @@ -60,16 +69,38 @@ def log_likelihood(calibrationData, prediction, numExperiments, covarianceMatrix # Check if the correct number of covariance terms has been passed in numResponses = len(edpLengthsList) if len(covarianceMatrixList) != numExperiments * numResponses: - print("ERROR: The expected number of covariance matrices is {}, but only {} were passed " - "in.".format(numExperiments * numResponses, len(covarianceMatrixList))) - raise CovError("ERROR: The expected number of covariance matrices is {}, but only {} were passed " - "in.".format(numExperiments * numResponses, len(covarianceMatrixList))) + print( + "ERROR: The expected number of covariance matrices is {}, but only {} were passed " + "in.".format( + numExperiments * numResponses, len(covarianceMatrixList) + ) + ) + raise CovError( + "ERROR: The expected number of covariance matrices is {}, but only {} were passed " + "in.".format( + numExperiments * numResponses, len(covarianceMatrixList) + ) + ) # Shift and normalize the prediction currentPosition = 0 for j in range(len(edpLengthsList)): - prediction[:, currentPosition:currentPosition + edpLengthsList[j]] = prediction[:, currentPosition:currentPosition + edpLengthsList[j]] + shiftFactors[j] - prediction[:, currentPosition:currentPosition + edpLengthsList[j]] = prediction[:, currentPosition:currentPosition + edpLengthsList[j]] / scaleFactors[j] + prediction[ + :, currentPosition : currentPosition + edpLengthsList[j] + ] = ( + prediction[ + :, currentPosition : currentPosition + edpLengthsList[j] + ] + + shiftFactors[j] + ) + prediction[ + :, currentPosition : currentPosition + edpLengthsList[j] + ] = ( + prediction[ + :, currentPosition : currentPosition + edpLengthsList[j] + ] + / scaleFactors[j] + ) currentPosition = currentPosition + edpLengthsList[j] # Compute the normalized residuals @@ -83,8 +114,10 @@ def log_likelihood(calibrationData, prediction, numExperiments, covarianceMatrix for j in range(numResponses): # Get the residuals corresponding to this response variable length = edpLengthsList[j] - residuals = allResiduals[i, currentPosition:currentPosition + length] - currentPosition = currentPosition + length + residuals = allResiduals[ + i, currentPosition : currentPosition + length + ] + currentPosition = currentPosition + length # Get the covariance matrix corresponding to this response variable cov = np.atleast_2d(covarianceMatrixList[covListIndex]) @@ -98,7 +131,11 @@ def log_likelihood(calibrationData, prediction, numExperiments, covarianceMatrix # having a sample of i.i.d. zero-mean normally distributed observations, and the log-likelihood can be # computed more efficiently var = cov[0][0] - ll = - length / 2 * np.log(var) - length / 2 * np.log(2 * np.pi) - 1 / (2 * var) * np.sum(residuals ** 2) + ll = ( + -length / 2 * np.log(var) + - length / 2 * np.log(2 * np.pi) + - 1 / (2 * var) * np.sum(residuals**2) + ) else: if np.shape(cov)[0] != np.shape(cov)[1]: cov = np.diag(cov.flatten()) @@ -109,7 +146,7 @@ def log_likelihood(calibrationData, prediction, numExperiments, covarianceMatrix t1 = length * np.log(2 * np.pi) eigenValues, eigenVectors = np.linalg.eigh(cov) logdet = np.sum(np.log(eigenValues)) - eigenValuesReciprocal = 1. / eigenValues + eigenValuesReciprocal = 1.0 / eigenValues z = eigenVectors * np.sqrt(eigenValuesReciprocal) mahalanobisDistance = np.square(np.dot(residuals, z)).sum() ll = -0.5 * (t1 + logdet + mahalanobisDistance) diff --git a/modules/performUQ/UCSD_UQ/loglike_script.py b/modules/performUQ/UCSD_UQ/loglike_script.py index 5be07baab..44c30add8 100644 --- a/modules/performUQ/UCSD_UQ/loglike_script.py +++ b/modules/performUQ/UCSD_UQ/loglike_script.py @@ -5,6 +5,7 @@ import numpy as np + def log_likelihood(residuals, mean, cov): length = len(residuals) if np.shape(cov)[0] == np.shape(cov)[1] == 1: @@ -12,7 +13,11 @@ def log_likelihood(residuals, mean, cov): # having a sample of i.i.d. zero-mean normally distributed observations, and the log-likelihood can be # computed more efficiently var = cov[0][0] - ll = - length / 2 * np.log(var) - length / 2 * np.log(2 * np.pi) - 1 / (2 * var) * np.sum(residuals ** 2) + ll = ( + -length / 2 * np.log(var) + - length / 2 * np.log(2 * np.pi) + - 1 / (2 * var) * np.sum(residuals**2) + ) else: if np.shape(cov)[0] != np.shape(cov)[1]: cov = np.diag(cov.flatten()) @@ -23,9 +28,9 @@ def log_likelihood(residuals, mean, cov): t1 = length * np.log(2 * np.pi) eigenValues, eigenVectors = np.linalg.eigh(cov) logdet = np.sum(np.log(eigenValues)) - eigenValuesReciprocal = 1. / eigenValues + eigenValuesReciprocal = 1.0 / eigenValues z = eigenVectors * np.sqrt(eigenValuesReciprocal) mahalanobisDistance = np.square(np.dot(residuals, z)).sum() ll = -0.5 * (t1 + logdet + mahalanobisDistance) - - return ll \ No newline at end of file + + return ll diff --git a/modules/performUQ/UCSD_UQ/mainscript.py b/modules/performUQ/UCSD_UQ/mainscript.py index fff3b705b..ab40498c6 100644 --- a/modules/performUQ/UCSD_UQ/mainscript.py +++ b/modules/performUQ/UCSD_UQ/mainscript.py @@ -39,7 +39,7 @@ def main(input_args): input_file_full_path = path_to_template_directory / input_file_name - with open(input_file_full_path, 'r', encoding='utf-8') as f: + with open(input_file_full_path, "r", encoding="utf-8") as f: inputs = json.load(f) uq_inputs = inputs["UQ"] diff --git a/modules/performUQ/UCSD_UQ/mainscript_hierarchical_bayesian.py b/modules/performUQ/UCSD_UQ/mainscript_hierarchical_bayesian.py index 246ab1264..a52f5223d 100644 --- a/modules/performUQ/UCSD_UQ/mainscript_hierarchical_bayesian.py +++ b/modules/performUQ/UCSD_UQ/mainscript_hierarchical_bayesian.py @@ -31,7 +31,7 @@ def generate_initial_states( if restart_file is not None: restart_file_path = Path(restart_file) - with restart_file_path.open(mode='r', encoding='utf-8') as f: + with restart_file_path.open(mode="r", encoding="utf-8") as f: restart_data = json.load(f) if "new_states" in restart_data: list_of_initial_states_of_model_parameters = [] @@ -79,7 +79,7 @@ def main(input_args): # input_file_full_path = template_directory / input_file - with open(input_file, 'r', encoding='utf-8') as f: + with open(input_file, "r", encoding="utf-8") as f: inputs = json.load(f) uq_inputs = inputs["UQ"] @@ -250,7 +250,9 @@ def main(input_args): list_of_initial_states_of_error_variance_per_dataset ) - with open(results_directory_path / f"sample_0.json", "w", encoding='utf-8') as f: + with open( + results_directory_path / f"sample_0.json", "w", encoding="utf-8" + ) as f: json.dump(results_to_write, f, indent=4) adaptivity_results = {} @@ -265,7 +267,9 @@ def main(input_args): cov_kernels_list ) with open( - results_directory_path.parent / f"adaptivity_results_{0}.json", "w", encoding='utf-8' + results_directory_path.parent / f"adaptivity_results_{0}.json", + "w", + encoding="utf-8", ) as f: json.dump(adaptivity_results, f, indent=4) @@ -302,6 +306,7 @@ def main(input_args): if run_type == "runningRemote": from mpi4py import MPI + MPI.COMM_WORLD.Abort(0) diff --git a/modules/performUQ/UCSD_UQ/mainscript_tmcmc.py b/modules/performUQ/UCSD_UQ/mainscript_tmcmc.py index 946280510..c80a9e3f6 100644 --- a/modules/performUQ/UCSD_UQ/mainscript_tmcmc.py +++ b/modules/performUQ/UCSD_UQ/mainscript_tmcmc.py @@ -3,6 +3,7 @@ affiliation: University of California, San Diego, *SimCenter, University of California, Berkeley """ + # ====================================================================================================================== import os import sys @@ -12,24 +13,47 @@ from parseData import parseDataFunction from runTMCMC import run_TMCMC -from calibration_utilities import CovarianceMatrixPreparer, CalDataPreparer, DataTransformer, createLogFile, syncLogFile, make_distributions, LogLikelihoodHandler +from calibration_utilities import ( + CovarianceMatrixPreparer, + CalDataPreparer, + DataTransformer, + createLogFile, + syncLogFile, + make_distributions, + LogLikelihoodHandler, +) # ====================================================================================================================== -def computeModelPosteriorProbabilities(modelPriorProbabilities, modelEvidences): + +def computeModelPosteriorProbabilities( + modelPriorProbabilities, modelEvidences +): denominator = np.dot(modelPriorProbabilities, modelEvidences) - return modelPriorProbabilities*modelEvidences/denominator + return modelPriorProbabilities * modelEvidences / denominator -def computeModelPosteriorProbabilitiesUsingLogEvidences(modelPriorProbabilities, modelLogEvidences): +def computeModelPosteriorProbabilitiesUsingLogEvidences( + modelPriorProbabilities, modelLogEvidences +): deltas = modelLogEvidences - np.min(modelLogEvidences) denominator = np.dot(modelPriorProbabilities, np.exp(deltas)) - return modelPriorProbabilities*np.exp(deltas)/denominator + return modelPriorProbabilities * np.exp(deltas) / denominator + # ====================================================================================================================== + class TMCMC_Data: - def __init__(self, mainscriptPath: str, workdirMain: str, runType: str, workflowDriver: str, logFile: TextIO, numBurnInSteps: int = 10) -> None: + def __init__( + self, + mainscriptPath: str, + workdirMain: str, + runType: str, + workflowDriver: str, + logFile: TextIO, + numBurnInSteps: int = 10, + ) -> None: self.mainscriptPath = mainscriptPath self.workdirMain = workdirMain self.runType = runType @@ -46,24 +70,27 @@ def __init__(self, mainscriptPath: str, workdirMain: str, runType: str, workflow def getMPI_size(self): if self.runType == "runningRemote": from mpi4py import MPI + self.comm = MPI.COMM_WORLD self.MPI_size = self.comm.Get_size() def updateUQInfo(self, numberOfSamples, seedVal): self.numberOfSamples = numberOfSamples self.seedVal = seedVal - + def findNumProcessorsAvailable(self): if self.runType == "runningLocal": import multiprocessing as mp + self.numProcessors = mp.cpu_count() elif self.runType == "runningRemote": from mpi4py import MPI + self.comm = MPI.COMM_WORLD self.numProcessors = self.comm.Get_size() else: self.numProcessors = 1 - + def getNumChains(self, numberOfSamples, runType, numProcessors): if runType == "runningLocal": self.numChains = int(min(numProcessors, self.recommendedNumChains)) @@ -71,16 +98,20 @@ def getNumChains(self, numberOfSamples, runType, numProcessors): self.numChains = int(max(numProcessors, self.recommendedNumChains)) else: self.numChains = self.recommendedNumChains - + if self.numChains < numberOfSamples: self.numChains = numberOfSamples - + def getNumStepsPerChainAfterBurnIn(self, numParticles, numChains): - self.numStepsAfterBurnIn = int(np.ceil(numParticles/numChains)) * self.numSkipSteps + self.numStepsAfterBurnIn = ( + int(np.ceil(numParticles / numChains)) * self.numSkipSteps + ) # self.numStepsPerChain = numBurnInSteps + numStepsAfterBurnIn + # ====================================================================================================================== + # ====================================================================================================================== def main(input_args): t1 = time.time() @@ -105,8 +136,8 @@ def main(input_args): # Remove dakotaTab and dakotaTabPrior files if they already exist in the working directory try: - os.remove('dakotaTab.out') - os.remove('dakotTabPrior.out') + os.remove("dakotaTab.out") + os.remove("dakotTabPrior.out") except OSError: pass @@ -116,62 +147,123 @@ def main(input_args): # input_json_filename_full_path = os.path.join(os.path.abspath(template_directory), input_json_filename) input_json_filename_full_path = input_json_filename logfile.write("\n\n==========================") - logfile.write("\nParsing the json input file {}".format(input_json_filename_full_path)) - (number_of_samples, seed_value, calibration_data_filename, loglikelihood_module, write_outputs, variables_list, - edp_names_list, edp_lengths_list, models_dict, total_number_of_models_in_ensemble) = parseDataFunction(input_json_filename_full_path, - logfile, working_directory, - os.path.dirname(mainscript_path)) + logfile.write( + "\nParsing the json input file {}".format( + input_json_filename_full_path + ) + ) + ( + number_of_samples, + seed_value, + calibration_data_filename, + loglikelihood_module, + write_outputs, + variables_list, + edp_names_list, + edp_lengths_list, + models_dict, + total_number_of_models_in_ensemble, + ) = parseDataFunction( + input_json_filename_full_path, + logfile, + working_directory, + os.path.dirname(mainscript_path), + ) syncLogFile(logfile) # # ================================================================================================================ # Initialize TMCMC object - tmcmc_data_instance = TMCMC_Data(mainscript_path, working_directory, run_type, driver_file, logfile, numBurnInSteps=4) - tmcmc_data_instance.updateUQInfo(number_of_samples, seed_value) - tmcmc_data_instance.findNumProcessorsAvailable() - tmcmc_data_instance.getNumChains(number_of_samples, run_type, tmcmc_data_instance.numProcessors) - tmcmc_data_instance.getNumStepsPerChainAfterBurnIn(number_of_samples, tmcmc_data_instance.numChains) + tmcmc_data_instance = TMCMC_Data( + mainscript_path, + working_directory, + run_type, + driver_file, + logfile, + numBurnInSteps=4, + ) + tmcmc_data_instance.updateUQInfo(number_of_samples, seed_value) + tmcmc_data_instance.findNumProcessorsAvailable() + tmcmc_data_instance.getNumChains( + number_of_samples, run_type, tmcmc_data_instance.numProcessors + ) + tmcmc_data_instance.getNumStepsPerChainAfterBurnIn( + number_of_samples, tmcmc_data_instance.numChains + ) # # ================================================================================================================ # Read calibration data - data_preparer_instance = CalDataPreparer(working_directory, template_directory, calibration_data_filename, - edp_names_list, edp_lengths_list, logfile) - calibration_data, number_of_experiments = data_preparer_instance.getCalibrationData() + data_preparer_instance = CalDataPreparer( + working_directory, + template_directory, + calibration_data_filename, + edp_names_list, + edp_lengths_list, + logfile, + ) + calibration_data, number_of_experiments = ( + data_preparer_instance.getCalibrationData() + ) # # ================================================================================================================ # Transform the data depending on the option chosen by the user transformation = "absMaxScaling" - data_transformer_instance = DataTransformer(transformStrategy=transformation, logFile=logfile) + data_transformer_instance = DataTransformer( + transformStrategy=transformation, logFile=logfile + ) - scale_factors, shift_factors = data_transformer_instance.computeScaleAndShiftFactors(calibration_data, edp_lengths_list) + scale_factors, shift_factors = ( + data_transformer_instance.computeScaleAndShiftFactors( + calibration_data, edp_lengths_list + ) + ) logfile.write("\n\n\tThe scale and shift factors computed are: ") for j in range(len(edp_names_list)): logfile.write( - "\n\t\tEDP: {}, scale factor: {}, shift factor: {}".format(edp_names_list[j], scale_factors[j], shift_factors[j]) + "\n\t\tEDP: {}, scale factor: {}, shift factor: {}".format( + edp_names_list[j], scale_factors[j], shift_factors[j] + ) ) transformed_calibration_data = data_transformer_instance.transformData() - logfile.write("\n\nThe transformed calibration data: \n{}".format(transformed_calibration_data)) + logfile.write( + "\n\nThe transformed calibration data: \n{}".format( + transformed_calibration_data + ) + ) # ====================================================================================================================== # Process covariance matrix options - cov_matrix_options_instance = CovarianceMatrixPreparer(transformed_calibration_data, edp_lengths_list, edp_names_list, - working_directory, number_of_experiments, logfile, run_type) - defaultErrorVariances = cov_matrix_options_instance.getDefaultErrorVariances() - covariance_matrix_list = cov_matrix_options_instance.createCovarianceMatrix() + cov_matrix_options_instance = CovarianceMatrixPreparer( + transformed_calibration_data, + edp_lengths_list, + edp_names_list, + working_directory, + number_of_experiments, + logfile, + run_type, + ) + defaultErrorVariances = ( + cov_matrix_options_instance.getDefaultErrorVariances() + ) + covariance_matrix_list = ( + cov_matrix_options_instance.createCovarianceMatrix() + ) # ====================================================================================================================== # Get log-likelihood function - LL_Handler = LogLikelihoodHandler(data=transformed_calibration_data, - covariance_matrix_blocks_list=covariance_matrix_list, - list_of_data_segment_lengths=edp_lengths_list, - list_of_scale_factors=scale_factors, - list_of_shift_factors=shift_factors, - workdir_main=working_directory, - full_path_to_tmcmc_code_directory=mainscript_path, - log_likelihood_file_name="loglike_script.py") + LL_Handler = LogLikelihoodHandler( + data=transformed_calibration_data, + covariance_matrix_blocks_list=covariance_matrix_list, + list_of_data_segment_lengths=edp_lengths_list, + list_of_scale_factors=scale_factors, + list_of_shift_factors=shift_factors, + workdir_main=working_directory, + full_path_to_tmcmc_code_directory=mainscript_path, + log_likelihood_file_name="loglike_script.py", + ) log_likelihood_function = LL_Handler.evaluate_log_likelihood # ====================================================================================================================== @@ -187,18 +279,29 @@ def main(input_args): logfile.write("\n\tNumber of particles: {}".format(number_of_samples)) # number of max MCMC steps - number_of_MCMC_steps = tmcmc_data_instance.numBurnInSteps + tmcmc_data_instance.numStepsAfterBurnIn + number_of_MCMC_steps = ( + tmcmc_data_instance.numBurnInSteps + + tmcmc_data_instance.numStepsAfterBurnIn + ) max_number_of_MCMC_steps = 10 - logfile.write("\n\tNumber of MCMC steps in first stage: {}".format(number_of_MCMC_steps)) logfile.write( - "\n\tMax. number of MCMC steps in any stage: {}".format(max_number_of_MCMC_steps) + "\n\tNumber of MCMC steps in first stage: {}".format( + number_of_MCMC_steps + ) + ) + logfile.write( + "\n\tMax. number of MCMC steps in any stage: {}".format( + max_number_of_MCMC_steps + ) ) syncLogFile(logfile) # ====================================================================================================================== # Initialize variables to store prior model probability and evidence - model_prior_probabilities = np.ones((len(variables_list),))/len(variables_list) + model_prior_probabilities = np.ones((len(variables_list),)) / len( + variables_list + ) model_evidences = np.ones_like(model_prior_probabilities) logfile.write("\n\n==========================") @@ -206,12 +309,18 @@ def main(input_args): # For each model: for model_number, parameters_of_model in enumerate(variables_list): logfile.write("\n\n\t==========================") - logfile.write("\n\tStarting analysis for model {}".format(model_number+1)) + logfile.write( + "\n\tStarting analysis for model {}".format(model_number + 1) + ) logfile.write("\n\t==========================") # Assign probability distributions to the parameters of the model - logfile.write("\n\t\tAssigning probability distributions to the parameters") - all_distributions_list = make_distributions(variables=parameters_of_model) + logfile.write( + "\n\t\tAssigning probability distributions to the parameters" + ) + all_distributions_list = make_distributions( + variables=parameters_of_model + ) # Run the Algorithm logfile.write("\n\n\t==========================") @@ -248,7 +357,7 @@ def main(input_args): driver_file, tmcmc_data_instance.parallelizeMCMC, model_number, - total_number_of_models_in_ensemble + total_number_of_models_in_ensemble, ) logfile.write("\n\n\t==========================") logfile.write("\n\tTMCMC algorithm finished running") @@ -290,12 +399,16 @@ def main(input_args): model_evidences[model_number] = evidence logfile.write("\n\n\t==========================") - logfile.write("\n\tCompleted analysis for model {}".format(model_number+1)) + logfile.write( + "\n\tCompleted analysis for model {}".format(model_number + 1) + ) logfile.write("\n\t==========================") syncLogFile(logfile) - modelPosteriorProbabilities = computeModelPosteriorProbabilities(model_prior_probabilities, model_evidences) + modelPosteriorProbabilities = computeModelPosteriorProbabilities( + model_prior_probabilities, model_evidences + ) logfile.write("\n\n==========================") logfile.write("\nFinished looping over each model") @@ -303,11 +416,15 @@ def main(input_args): logfile.write("\nThe posterior model probabilities are:") for model_number in range(len(variables_list)): - logfile.write(f"\nModel number {model_number+1}: {modelPosteriorProbabilities[model_number]*100:15g}%") + logfile.write( + f"\nModel number {model_number+1}: {modelPosteriorProbabilities[model_number]*100:15g}%" + ) # ====================================================================================================================== logfile.write("\nUCSD_UQ engine workflow complete!\n") - logfile.write("\nTime taken: {:0.2f} minutes\n\n".format((time.time() - t1) / 60)) + logfile.write( + "\nTime taken: {:0.2f} minutes\n\n".format((time.time() - t1) / 60) + ) syncLogFile(logfile) @@ -325,4 +442,4 @@ def main(input_args): inputArgs = sys.argv main(inputArgs) -# ====================================================================================================================== \ No newline at end of file +# ====================================================================================================================== diff --git a/modules/performUQ/UCSD_UQ/parseData.py b/modules/performUQ/UCSD_UQ/parseData.py index b12f43270..d39b36e6b 100644 --- a/modules/performUQ/UCSD_UQ/parseData.py +++ b/modules/performUQ/UCSD_UQ/parseData.py @@ -14,6 +14,7 @@ import numpy as np import itertools + class DataProcessingError(Exception): """Raised when errors found when processing user-supplied calibration and covariance data. @@ -58,7 +59,7 @@ def parseDataFunction(dakotaJsonFile, logFile, tmpSimCenterDir, mainscriptDir): logFile.write("\n\t\tProcessing UQ inputs") seedValue = uqInputs["seed"] nSamples = uqInputs["numParticles"] - #maxRunTime = uqInputs["maxRunTime"] + # maxRunTime = uqInputs["maxRunTime"] if "maxRunTime" in uqInputs.keys(): maxRunTime = uqInputs["maxRunTime"] else: @@ -72,7 +73,9 @@ def parseDataFunction(dakotaJsonFile, logFile, tmpSimCenterDir, mainscriptDir): logFile.write("\n\t\t\tProcessing the log-likelihood script options") # If log-likelihood script is provided, use that, otherwise, use default log-likelihood function - if len(logLikelihoodFile) > 0: # if the log-likelihood file is not an empty string + if ( + len(logLikelihoodFile) > 0 + ): # if the log-likelihood file is not an empty string logFile.write( "\n\t\t\t\tSearching for a user-defined log-likelihood script '{}'".format( logLikelihoodFile @@ -113,7 +116,9 @@ def parseDataFunction(dakotaJsonFile, logFile, tmpSimCenterDir, mainscriptDir): logFile.write("\n\t\t\t\tLog-likelihood script not provided.") logFile.write( "\n\t\t\t\tUsing the default log-likelihood script: \n\t\t\t\t\t{}".format( - os.path.join(defaultLogLikeDirectoryPath, defaultLogLikeFileName) + os.path.join( + defaultLogLikeDirectoryPath, defaultLogLikeFileName + ) ) ) try: @@ -153,7 +158,7 @@ def parseDataFunction(dakotaJsonFile, logFile, tmpSimCenterDir, mainscriptDir): modelsDict = {} modelIndicesList = [] modelRVNamesList = [] - applications = jsonInputs["Applications"] + applications = jsonInputs["Applications"] for app, appInputs in applications.items(): logFile.write(f"\n\t\t\tApp: {app}") if app.lower() not in ["events"]: @@ -162,24 +167,26 @@ def parseDataFunction(dakotaJsonFile, logFile, tmpSimCenterDir, mainscriptDir): appl = appInputs[0]["Application"].lower() if appl in ["multimodel"]: # runMultiModel = True - logFile.write(f'\n\t\t\t\tFound a multimodel application - {app}: {appInputs["Application"]}') + logFile.write( + f'\n\t\t\t\tFound a multimodel application - {app}: {appInputs["Application"]}' + ) modelRVName = jsonInputs[app]["modelToRun"][3:] appModels = jsonInputs[app]["models"] nM = len(appModels) - logFile.write(f'\n\t\t\t\t\tThere are {nM} {app} models') + logFile.write(f"\n\t\t\t\t\tThere are {nM} {app} models") modelData = {} modelData["nModels"] = nM - modelData["values"] = [i+1 for i in range(nM)] + modelData["values"] = [i + 1 for i in range(nM)] modelData["weights"] = [model["belief"] for model in appModels] modelData["name"] = modelRVName modelsDict[app] = modelData modelIndicesList.append(modelData["values"]) modelRVNamesList.append(modelRVName) else: - logFile.write('\n\t\t\t\tNot a multimodel application') + logFile.write("\n\t\t\t\tNot a multimodel application") nModels = 1 for _, data in modelsDict.items(): - nModels = nModels*data["nModels"] + nModels = nModels * data["nModels"] cartesianProductOfModelIndices = list(itertools.product(*modelIndicesList)) # logFile.write("\n\t\t\tNO LONGER Getting the number of models") # inputFileList = [] @@ -197,20 +204,24 @@ def parseDataFunction(dakotaJsonFile, logFile, tmpSimCenterDir, mainscriptDir): # Variables variablesList = [] for _ in range(nModels): - variablesList.append({ - "names": [], - "distributions": [], - "Par1": [], - "Par2": [], - "Par3": [], - "Par4": [], - }) + variablesList.append( + { + "names": [], + "distributions": [], + "Par1": [], + "Par2": [], + "Par3": [], + "Par4": [], + } + ) logFile.write("\n\n\t\t\tLooping over the models") for ind in range(nModels): logFile.write("\n\t\t\t\tModel number: {}".format(ind)) # Processing RV inputs - logFile.write("\n\t\t\t\t\tCreating priors for model number {}".format(ind)) + logFile.write( + "\n\t\t\t\t\tCreating priors for model number {}".format(ind) + ) logFile.write("\n\t\t\t\t\t\tProcessing RV inputs") for i, rv in enumerate(rvInputs): variablesList[ind]["names"].append(rv["name"]) @@ -252,7 +263,10 @@ def parseDataFunction(dakotaJsonFile, logFile, tmpSimCenterDir, mainscriptDir): variablesList[ind]["Par3"].append(rv["lowerbound"]) variablesList[ind]["Par4"].append(rv["upperbound"]) paramString = "params: {}, {}, {}, {}".format( - rv["alphas"], rv["betas"], rv["lowerbound"], rv["upperbound"] + rv["alphas"], + rv["betas"], + rv["lowerbound"], + rv["upperbound"], ) elif rv["distribution"] == "Lognormal": # meanValue = rv["mean"] @@ -273,7 +287,9 @@ def parseDataFunction(dakotaJsonFile, logFile, tmpSimCenterDir, mainscriptDir): variablesList[ind]["Par2"].append(rv["betaparam"]) variablesList[ind]["Par3"].append(None) variablesList[ind]["Par4"].append(None) - paramString = "params: {}, {}".format(rv["alphaparam"], rv["betaparam"]) + paramString = "params: {}, {}".format( + rv["alphaparam"], rv["betaparam"] + ) elif rv["distribution"] == "Weibull": variablesList[ind]["Par1"].append(rv["shapeparam"]) variablesList[ind]["Par2"].append(rv["scaleparam"]) @@ -312,7 +328,9 @@ def parseDataFunction(dakotaJsonFile, logFile, tmpSimCenterDir, mainscriptDir): if "multimodel" in rv["name"].lower(): try: index = modelRVNamesList.index(rv["name"]) - variablesList[ind]["Par1"].append(cartesianProductOfModelIndices[ind][index]) + variablesList[ind]["Par1"].append( + cartesianProductOfModelIndices[ind][index] + ) variablesList[ind]["Par2"].append(None) variablesList[ind]["Par3"].append(None) variablesList[ind]["Par4"].append(None) @@ -320,8 +338,10 @@ def parseDataFunction(dakotaJsonFile, logFile, tmpSimCenterDir, mainscriptDir): cartesianProductOfModelIndices[ind][index] ) except ValueError: - logFile.write(f"{rv['name']} not found in list of model RV names") - + logFile.write( + f"{rv['name']} not found in list of model RV names" + ) + else: variablesList[ind]["Par1"].append(rv["Values"]) variablesList[ind]["Par2"].append(rv["Weights"]) @@ -366,7 +386,7 @@ def parseDataFunction(dakotaJsonFile, logFile, tmpSimCenterDir, mainscriptDir): i, name, "InvGamma", paramString ) ) - + logFile.write("\n\n\tCompleted parsing the inputs") logFile.write("\n\n==========================") logFile.flush() @@ -381,5 +401,5 @@ def parseDataFunction(dakotaJsonFile, logFile, tmpSimCenterDir, mainscriptDir): edpNamesList, edpLengthsList, modelsDict, - nModels + nModels, ) diff --git a/modules/performUQ/UCSD_UQ/pdfs.py b/modules/performUQ/UCSD_UQ/pdfs.py index d49ae6004..abfc067fd 100644 --- a/modules/performUQ/UCSD_UQ/pdfs.py +++ b/modules/performUQ/UCSD_UQ/pdfs.py @@ -141,7 +141,9 @@ def __init__(self, alpha, beta, lowerbound, upperbound): self.beta = beta self.lowerbound = lowerbound self.upperbound = upperbound - self.dist = stats.beta(self.alpha, self.beta, self.lowerbound, self.upperbound) + self.dist = stats.beta( + self.alpha, self.beta, self.lowerbound, self.upperbound + ) def generate_rns(self, N): return self.dist.rvs(size=N) @@ -255,7 +257,9 @@ def __init__(self, values, weights): self.values = values self.weights = weights self.probabilities = self.weights / np.sum(self.weights) - self.log_probabilities = np.log(self.weights) - np.log(np.sum(self.weights)) + self.log_probabilities = np.log(self.weights) - np.log( + np.sum(self.weights) + ) self.rng = np.random.default_rng() def generate_rns(self, N): @@ -278,12 +282,13 @@ def log_pdf_eval(self, u): lp[i] = self.log_probabilities[np.where(self.values == x_comp)] return lp + class ConstantInteger: def __init__(self, value) -> None: self.value = value - + def generate_rns(self, N): return np.array([self.value for _ in range(N)], dtype=int) - + def log_pdf_eval(self, x): return 0.0 diff --git a/modules/performUQ/UCSD_UQ/processInputs.py b/modules/performUQ/UCSD_UQ/processInputs.py index 126898e27..e210f6dd0 100644 --- a/modules/performUQ/UCSD_UQ/processInputs.py +++ b/modules/performUQ/UCSD_UQ/processInputs.py @@ -6,39 +6,42 @@ import platform import argparse -if __name__ == '__main__': +if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument('--workflowInput') - parser.add_argument('--workflowOutput') - parser.add_argument('--driverFile') - parser.add_argument('--runType') + parser.add_argument("--workflowInput") + parser.add_argument("--workflowOutput") + parser.add_argument("--driverFile") + parser.add_argument("--runType") - args,unknowns = parser.parse_known_args() + args, unknowns = parser.parse_known_args() inputFile = args.workflowInput runType = args.runType workflowDriver = args.driverFile - outputFile = args.workflowOutput + outputFile = args.workflowOutput cwd = os.getcwd() - workdir_main = str(Path(cwd).parents[0]) - - #mainScriptPath = inputArgs[0] - #tmpSimCenterDir = inputArgs[1] - #templateDir = inputArgs[2] - #runType = inputArgs[3] # either "runningLocal" or "runningRemote" + workdir_main = str(Path(cwd).parents[0]) + + # mainScriptPath = inputArgs[0] + # tmpSimCenterDir = inputArgs[1] + # templateDir = inputArgs[2] + # runType = inputArgs[3] # either "runningLocal" or "runningRemote" mainScriptPath = os.path.dirname(os.path.realpath(__file__)) templateDir = cwd - tmpSimCenterDir = str(Path(cwd).parents[0]) - + tmpSimCenterDir = str(Path(cwd).parents[0]) + # Change permission of workflow driver if platform.system() != "Windows": workflowDriverFile = os.path.join(templateDir, workflowDriver) - if runType in ['runningLocal']: - os.chmod(workflowDriverFile, stat.S_IWUSR | stat.S_IXUSR | stat.S_IRUSR | stat.S_IXOTH) + if runType in ["runningLocal"]: + os.chmod( + workflowDriverFile, + stat.S_IWUSR | stat.S_IXUSR | stat.S_IRUSR | stat.S_IXOTH, + ) st = os.stat(workflowDriverFile) os.chmod(workflowDriverFile, st.st_mode | stat.S_IEXEC) pythonCommand = "python3" @@ -50,7 +53,7 @@ if runType in ["runningLocal"]: # Get path to python from dakota.json file dakotaJsonFile = os.path.join(os.path.abspath(templateDir), inputFile) - with open(dakotaJsonFile, 'r') as f: + with open(dakotaJsonFile, "r") as f: jsonInputs = json.load(f) if "python" in jsonInputs.keys(): @@ -59,11 +62,20 @@ # Get the path to the mainscript.py of TMCMC # mainScriptDir = os.path.split(mainScriptPath)[0] mainScript = os.path.join(mainScriptPath, "mainscript.py") - command = "{} {} {} {} {} {} {}".format(pythonCommand, mainScript, tmpSimCenterDir, templateDir, runType, workflowDriver, inputFile) + command = "{} {} {} {} {} {} {}".format( + pythonCommand, + mainScript, + tmpSimCenterDir, + templateDir, + runType, + workflowDriver, + inputFile, + ) try: - result = subprocess.check_output(command, stderr=subprocess.STDOUT, shell=True) + result = subprocess.check_output( + command, stderr=subprocess.STDOUT, shell=True + ) returnCode = 0 except subprocess.CalledProcessError as e: result = e.output returnCode = e.returncode - diff --git a/modules/performUQ/UCSD_UQ/runFEM.py b/modules/performUQ/UCSD_UQ/runFEM.py index 8c1fc1be8..bb3f23f22 100644 --- a/modules/performUQ/UCSD_UQ/runFEM.py +++ b/modules/performUQ/UCSD_UQ/runFEM.py @@ -21,29 +21,47 @@ def copytree(src, dst, symlinks=False, ignore=None): copytree(s, d, symlinks, ignore) else: try: - if not os.path.exists(d) or os.stat(s).st_mtime - os.stat(d).st_mtime > 1: + if ( + not os.path.exists(d) + or os.stat(s).st_mtime - os.stat(d).st_mtime > 1 + ): shutil.copy2(s, d) except Exception as ex: - msg = f"Could not copy {s}. The following error occurred: \n{ex}" + msg = ( + f"Could not copy {s}. The following error occurred: \n{ex}" + ) return msg return "0" -def runFEM(particleNumber, parameterSampleValues, variables, workdirMain, log_likelihood_function, calibrationData, numExperiments, - covarianceMatrixList, edpNamesList, edpLengthsList, scaleFactors, shiftFactors, workflowDriver): - """ +def runFEM( + particleNumber, + parameterSampleValues, + variables, + workdirMain, + log_likelihood_function, + calibrationData, + numExperiments, + covarianceMatrixList, + edpNamesList, + edpLengthsList, + scaleFactors, + shiftFactors, + workflowDriver, +): + """ this function runs FE model (model.tcl) for each parameter value (par) model.tcl should take parameter input model.tcl should output 'output$PN.txt' -> column vector of size 'Ny' """ - workdirName = ("workdir." + str(particleNumber + 1)) + workdirName = "workdir." + str(particleNumber + 1) analysisPath = os.path.join(workdirMain, workdirName) if os.path.isdir(analysisPath): os.chmod(os.path.join(analysisPath, workflowDriver), 0o777) shutil.rmtree(analysisPath) - + os.mkdir(analysisPath) # copy templatefiles @@ -57,36 +75,34 @@ def runFEM(particleNumber, parameterSampleValues, variables, workdirMain, log_li covarianceMultiplierList = [] parameterNames = variables["names"] with open("params.in", "w") as f: - f.write('{}\n'.format(len(parameterSampleValues) - len(edpNamesList))) + f.write("{}\n".format(len(parameterSampleValues) - len(edpNamesList))) for i in range(len(parameterSampleValues)): name = str(parameterNames[i]) value = str(parameterSampleValues[i]) - if name.split('.')[-1] != 'CovMultiplier': - f.write('{} {}\n'.format(name, value)) + if name.split(".")[-1] != "CovMultiplier": + f.write("{} {}\n".format(name, value)) else: covarianceMultiplierList.append(parameterSampleValues[i]) - #subprocess.run(workflowDriver, stderr=subprocess.PIPE, shell=True) + # subprocess.run(workflowDriver, stderr=subprocess.PIPE, shell=True) returnCode = subprocess.call( - os.path.join(analysisPath, workflowDriver), - shell=True, - stdout=subprocess.DEVNULL, - stderr=subprocess.STDOUT, - ) # subprocess.check_call(workflow_run_command, shell=True, stdout=FNULL, stderr=subprocess.STDOUT) - - + os.path.join(analysisPath, workflowDriver), + shell=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.STDOUT, + ) # subprocess.check_call(workflow_run_command, shell=True, stdout=FNULL, stderr=subprocess.STDOUT) # Read in the model prediction - if os.path.exists('results.out'): - with open('results.out', 'r') as f: + if os.path.exists("results.out"): + with open("results.out", "r") as f: prediction = np.atleast_2d(np.genfromtxt(f)).reshape((1, -1)) preds = prediction.copy() os.chdir("../") ll = log_likelihood_function(prediction, covarianceMultiplierList) else: os.chdir("../") - preds = np.atleast_2d([-np.inf]*sum(edpLengthsList)).reshape((1, -1)) + preds = np.atleast_2d([-np.inf] * sum(edpLengthsList)).reshape((1, -1)) ll = -np.inf - return (ll, preds) \ No newline at end of file + return (ll, preds) diff --git a/modules/performUQ/UCSD_UQ/runTMCMC.py b/modules/performUQ/UCSD_UQ/runTMCMC.py index f07bd561e..92f8021ce 100644 --- a/modules/performUQ/UCSD_UQ/runTMCMC.py +++ b/modules/performUQ/UCSD_UQ/runTMCMC.py @@ -1,5 +1,5 @@ """ -authors: Mukesh Kumar Ramancha, Maitreya Manoj Kurumbhati, and Prof. J.P. Conte +authors: Mukesh Kumar Ramancha, Maitreya Manoj Kurumbhati, and Prof. J.P. Conte affiliation: University of California, San Diego modified: Aakash Bangalore Satish, NHERI SimCenter, UC Berkeley """ @@ -14,9 +14,16 @@ import csv -def write_stage_start_info_to_logfile(logfile, stage_number, beta, effective_sample_size, - scale_factor_for_proposal_covariance, log_evidence, number_of_samples): - logfile.write('\n\n\t\t==========================') +def write_stage_start_info_to_logfile( + logfile, + stage_number, + beta, + effective_sample_size, + scale_factor_for_proposal_covariance, + log_evidence, + number_of_samples, +): + logfile.write("\n\n\t\t==========================") logfile.write("\n\t\tStage number: {}".format(stage_number)) if stage_number == 0: logfile.write("\n\t\tSampling from prior") @@ -26,23 +33,42 @@ def write_stage_start_info_to_logfile(logfile, stage_number, beta, effective_sam logfile.write("\n\t\tESS = %d" % effective_sample_size) logfile.write("\n\t\tscalem = %.2g" % scale_factor_for_proposal_covariance) logfile.write(f"\n\t\tlog-evidence = {log_evidence:<9.8g}") - logfile.write("\n\n\t\tNumber of model evaluations in this stage: {}".format(number_of_samples)) + logfile.write( + "\n\n\t\tNumber of model evaluations in this stage: {}".format( + number_of_samples + ) + ) logfile.flush() os.fsync(logfile.fileno()) -def write_eval_data_to_logfile(logfile, parallelize_MCMC, run_type, proc_count=1, MPI_size=1, stage_num=0): + +def write_eval_data_to_logfile( + logfile, parallelize_MCMC, run_type, proc_count=1, MPI_size=1, stage_num=0 +): if stage_num == 0: logfile.write("\n\n\t\tRun type: {}".format(run_type)) if parallelize_MCMC: if run_type == "runningLocal": if stage_num == 0: - logfile.write("\n\n\t\tCreated multiprocessing pool for runType: {}".format(run_type)) + logfile.write( + "\n\n\t\tCreated multiprocessing pool for runType: {}".format( + run_type + ) + ) else: logfile.write("\n\n\t\tLocal run - MCMC steps") - logfile.write("\n\t\t\tNumber of processors being used: {}".format(proc_count)) + logfile.write( + "\n\t\t\tNumber of processors being used: {}".format( + proc_count + ) + ) else: if stage_num == 0: - logfile.write("\n\n\t\tCreated mpi4py executor pool for runType: {}".format(run_type)) + logfile.write( + "\n\n\t\tCreated mpi4py executor pool for runType: {}".format( + run_type + ) + ) else: logfile.write("\n\n\t\tRemote run - MCMC steps") logfile.write("\n\t\t\tmax_workers: {}".format(MPI_size)) @@ -54,47 +80,75 @@ def write_eval_data_to_logfile(logfile, parallelize_MCMC, run_type, proc_count=1 logfile.write("\n\t\t\tNumber of processors being used: {}".format(1)) -def create_headings(logfile, model_number, model_parameters, edp_names_list, edp_lengths_list, writeOutputs): +def create_headings( + logfile, + model_number, + model_parameters, + edp_names_list, + edp_lengths_list, + writeOutputs, +): # Create the headings, which will be the first line of the file - headings = 'eval_id\tinterface\t' + headings = "eval_id\tinterface\t" if model_number == 0: logfile.write("\n\t\t\tCreating headings") - for v in model_parameters['names']: - headings += '{}\t'.format(v) + for v in model_parameters["names"]: + headings += "{}\t".format(v) if writeOutputs: # create headings for outputs for i, edp in enumerate(edp_names_list): if edp_lengths_list[i] == 1: - headings += '{}\t'.format(edp) + headings += "{}\t".format(edp) else: for comp in range(edp_lengths_list[i]): - headings += '{}_{}\t'.format(edp, comp + 1) - headings += '\n' - + headings += "{}_{}\t".format(edp, comp + 1) + headings += "\n" + return headings def get_prediction_from_workdirs(i, working_directory): - workdir_string = ("workdir." + str(i + 1)) - prediction = np.atleast_2d(np.genfromtxt(os.path.join(working_directory, workdir_string, - 'results.out'))).reshape((1, -1)) + workdir_string = "workdir." + str(i + 1) + prediction = np.atleast_2d( + np.genfromtxt( + os.path.join(working_directory, workdir_string, "results.out") + ) + ).reshape((1, -1)) return prediction -def write_data_to_tab_files(logfile, working_directory, model_number, model_parameters, - edp_names_list, edp_lengths_list, number_of_samples, dataToWrite, - tab_file_name, predictions): - +def write_data_to_tab_files( + logfile, + working_directory, + model_number, + model_parameters, + edp_names_list, + edp_lengths_list, + number_of_samples, + dataToWrite, + tab_file_name, + predictions, +): + tab_file_full_path = os.path.join(working_directory, tab_file_name) write_outputs = True - headings = create_headings(logfile, model_number, model_parameters, edp_names_list, edp_lengths_list, write_outputs) + headings = create_headings( + logfile, + model_number, + model_parameters, + edp_names_list, + edp_lengths_list, + write_outputs, + ) logfile.write("\n\t\t\tWriting to file {}".format(tab_file_full_path)) with open(tab_file_full_path, "a+") as f: if model_number == 0: f.write(headings) for i in range(number_of_samples): - row_string = f"{i + 1 + number_of_samples*model_number}\t{model_number+1}\t" - for j in range(len(model_parameters['names'])): + row_string = ( + f"{i + 1 + number_of_samples*model_number}\t{model_number+1}\t" + ) + for j in range(len(model_parameters["names"])): row_string += f"{dataToWrite[i, j]}\t" if write_outputs: # write the output data prediction = predictions[i, :] @@ -103,33 +157,67 @@ def write_data_to_tab_files(logfile, working_directory, model_number, model_para row_string += "\n" f.write(row_string) - logfile.write('\n\t\t==========================') + logfile.write("\n\t\t==========================") logfile.flush() os.fsync(logfile.fileno()) -def write_data_to_csvfile(logfile, total_number_of_models_in_ensemble, stage_number, model_number, - working_directory, data_to_write): - logfile.write("\n\n\t\tWriting samples from stage {} to csv file".format(stage_number - 1)) +def write_data_to_csvfile( + logfile, + total_number_of_models_in_ensemble, + stage_number, + model_number, + working_directory, + data_to_write, +): + logfile.write( + "\n\n\t\tWriting samples from stage {} to csv file".format( + stage_number - 1 + ) + ) if total_number_of_models_in_ensemble > 1: - string_to_append = f'resultsStage{stage_number - 1}_Model_{model_number+1}.csv' + string_to_append = ( + f"resultsStage{stage_number - 1}_Model_{model_number+1}.csv" + ) else: - string_to_append = f'resultsStage{stage_number - 1}.csv' - resultsFilePath = os.path.join(os.path.abspath(working_directory), string_to_append) + string_to_append = f"resultsStage{stage_number - 1}.csv" + resultsFilePath = os.path.join( + os.path.abspath(working_directory), string_to_append + ) - with open(resultsFilePath, 'w', newline='') as csvfile: + with open(resultsFilePath, "w", newline="") as csvfile: csvWriter = csv.writer(csvfile) csvWriter.writerows(data_to_write) logfile.write("\n\t\t\tWrote to file {}".format(resultsFilePath)) # Finished writing data -def run_TMCMC(number_of_samples, number_of_chains, all_distributions_list, number_of_MCMC_steps, max_number_of_MCMC_steps, - log_likelihood_function, model_parameters, working_directory, seed, - calibration_data, number_of_experiments, covariance_matrix_list, edp_names_list, edp_lengths_list, scale_factors, - shift_factors, run_type, logfile, MPI_size, driver_file, parallelize_MCMC=True, - model_number=0, total_number_of_models_in_ensemble=1): - """ Runs TMCMC Algorithm """ +def run_TMCMC( + number_of_samples, + number_of_chains, + all_distributions_list, + number_of_MCMC_steps, + max_number_of_MCMC_steps, + log_likelihood_function, + model_parameters, + working_directory, + seed, + calibration_data, + number_of_experiments, + covariance_matrix_list, + edp_names_list, + edp_lengths_list, + scale_factors, + shift_factors, + run_type, + logfile, + MPI_size, + driver_file, + parallelize_MCMC=True, + model_number=0, + total_number_of_models_in_ensemble=1, +): + """Runs TMCMC Algorithm""" # Initialize (beta, effective sample size) beta = 0 @@ -139,30 +227,67 @@ def run_TMCMC(number_of_samples, number_of_chains, all_distributions_list, numbe # Initialize other TMCMC variables number_of_MCMC_steps = number_of_MCMC_steps adaptively_calculate_num_MCMC_steps = True - adaptively_scale_proposal_covariance = True + adaptively_scale_proposal_covariance = True scale_factor_for_proposal_covariance = 1 # cov scale factor # model_evidence = 1 # model evidence stage_number = 0 # stage number of TMCMC log_evidence = 0 - write_stage_start_info_to_logfile(logfile, stage_number, beta, effective_sample_size, - scale_factor_for_proposal_covariance, log_evidence, number_of_samples) + write_stage_start_info_to_logfile( + logfile, + stage_number, + beta, + effective_sample_size, + scale_factor_for_proposal_covariance, + log_evidence, + number_of_samples, + ) # initial samples - sample_values = tmcmcFunctions.initial_population(number_of_samples, all_distributions_list) + sample_values = tmcmcFunctions.initial_population( + number_of_samples, all_distributions_list + ) # Evaluate posterior at Sm - prior_pdf_values = np.array([tmcmcFunctions.log_prior(s, all_distributions_list) for s in sample_values]).squeeze() - unnormalized_posterior_pdf_values = prior_pdf_values # prior = post for beta = 0 - - iterables = [(ind, sample_values[ind], model_parameters, working_directory, log_likelihood_function, calibration_data, - number_of_experiments, covariance_matrix_list, edp_names_list, edp_lengths_list, - scale_factors, shift_factors, driver_file) for ind in range(number_of_samples)] + prior_pdf_values = np.array( + [ + tmcmcFunctions.log_prior(s, all_distributions_list) + for s in sample_values + ] + ).squeeze() + unnormalized_posterior_pdf_values = ( + prior_pdf_values # prior = post for beta = 0 + ) + + iterables = [ + ( + ind, + sample_values[ind], + model_parameters, + working_directory, + log_likelihood_function, + calibration_data, + number_of_experiments, + covariance_matrix_list, + edp_names_list, + edp_lengths_list, + scale_factors, + shift_factors, + driver_file, + ) + for ind in range(number_of_samples) + ] # Evaluate log-likelihood at current samples Sm if run_type == "runningLocal": processor_count = mp.cpu_count() pool = Pool(processes=processor_count) - write_eval_data_to_logfile(logfile, parallelize_MCMC, run_type, proc_count=processor_count, stage_num=stage_number) + write_eval_data_to_logfile( + logfile, + parallelize_MCMC, + run_type, + proc_count=processor_count, + stage_num=stage_number, + ) outputs = pool.starmap(runFEM, iterables) log_likelihoods_list = [] predictions_list = [] @@ -171,8 +296,15 @@ def run_TMCMC(number_of_samples, number_of_chains, all_distributions_list, numbe predictions_list.append(output[1]) else: from mpi4py.futures import MPIPoolExecutor + executor = MPIPoolExecutor(max_workers=MPI_size) - write_eval_data_to_logfile(logfile, parallelize_MCMC, run_type, MPI_size=MPI_size, stage_num=stage_number) + write_eval_data_to_logfile( + logfile, + parallelize_MCMC, + run_type, + MPI_size=MPI_size, + stage_num=stage_number, + ) outputs = list(executor.starmap(runFEM, iterables)) log_likelihoods_list = [] predictions_list = [] @@ -180,16 +312,33 @@ def run_TMCMC(number_of_samples, number_of_chains, all_distributions_list, numbe log_likelihoods_list.append(output[0]) predictions_list.append(output[1]) log_likelihood_values = np.array(log_likelihoods_list).squeeze() - prediction_values = np.array(predictions_list).reshape((number_of_samples, -1)) + prediction_values = np.array(predictions_list).reshape( + (number_of_samples, -1) + ) total_number_of_model_evaluations = number_of_samples - logfile.write("\n\n\t\tTotal number of model evaluations so far: {}".format(total_number_of_model_evaluations)) + logfile.write( + "\n\n\t\tTotal number of model evaluations so far: {}".format( + total_number_of_model_evaluations + ) + ) # Write the results of the first stage to a file named dakotaTabPrior.out for quoFEM to be able to read the results - logfile.write("\n\n\t\tWriting prior samples to 'dakotaTabPrior.out' for quoFEM to read the results") - write_data_to_tab_files(logfile, working_directory, model_number, model_parameters, - edp_names_list, edp_lengths_list, number_of_samples, dataToWrite=sample_values, - tab_file_name="dakotaTabPrior.out", predictions=prediction_values) + logfile.write( + "\n\n\t\tWriting prior samples to 'dakotaTabPrior.out' for quoFEM to read the results" + ) + write_data_to_tab_files( + logfile, + working_directory, + model_number, + model_parameters, + edp_names_list, + edp_lengths_list, + number_of_samples, + dataToWrite=sample_values, + tab_file_name="dakotaTabPrior.out", + predictions=prediction_values, + ) total_log_evidence = 0 @@ -199,7 +348,11 @@ def run_TMCMC(number_of_samples, number_of_chains, all_distributions_list, numbe # plausible weights of Sm corresponding to new beta # beta, Wm, ESS = tmcmcFunctions.compute_beta(beta, Lm, ESS, threshold=0.95) # beta, Wm, ESS = tmcmcFunctions.compute_beta(beta, Lm, ESS, threshold=0.5) - beta, log_evidence, weights, effective_sample_size = tmcmcFunctions.compute_beta_evidence(beta, log_likelihood_values, logfile, threshold=1.0) + beta, log_evidence, weights, effective_sample_size = ( + tmcmcFunctions.compute_beta_evidence( + beta, log_likelihood_values, logfile, threshold=1.0 + ) + ) # beta, log_evidence, weights, effective_sample_size = tmcmcFunctions.compute_beta_evidence_old(beta, log_likelihood_values, logfile, int(effective_sample_size/2), threshold=1.0) total_log_evidence = total_log_evidence + log_evidence @@ -212,104 +365,231 @@ def run_TMCMC(number_of_samples, number_of_chains, all_distributions_list, numbe # model_evidence = model_evidence * (sum(weights) / number_of_samples) # Calculate covariance matrix using Wm_n - weighted_sample_covariance_matrix = np.cov(sample_values, aweights=weights, rowvar=False) + weighted_sample_covariance_matrix = np.cov( + sample_values, aweights=weights, rowvar=False + ) # logFile.write("\nCovariance matrix: {}".format(Cm)) # Resample ################################################### # Resampling using plausible weights # SmcapIDs = np.random.choice(range(N), N, p=Wm / sum(Wm)) rng = default_rng(child_seeds[-1]) - resample_ids = rng.choice(range(number_of_samples), number_of_samples, p=weights) + resample_ids = rng.choice( + range(number_of_samples), number_of_samples, p=weights + ) resampled_values = sample_values[resample_ids] resampled_log_likelihood_values = log_likelihood_values[resample_ids] - resampled_unnormalized_posterior_pdf_values = unnormalized_posterior_pdf_values[resample_ids] - resampled_prediction_values = np.atleast_2d(prediction_values[resample_ids, :]) + resampled_unnormalized_posterior_pdf_values = ( + unnormalized_posterior_pdf_values[resample_ids] + ) + resampled_prediction_values = np.atleast_2d( + prediction_values[resample_ids, :] + ) # save to trace # stage m: samples, likelihood, weights, next stage ESS, next stage beta, resampled samples - mytrace.append([sample_values, log_likelihood_values, weights, effective_sample_size, beta, resampled_values]) + mytrace.append( + [ + sample_values, + log_likelihood_values, + weights, + effective_sample_size, + beta, + resampled_values, + ] + ) # Write Data to '.csv' files data_to_write = np.hstack((sample_values, prediction_values)) - write_data_to_csvfile(logfile, total_number_of_models_in_ensemble, stage_number, model_number, - working_directory, data_to_write) + write_data_to_csvfile( + logfile, + total_number_of_models_in_ensemble, + stage_number, + model_number, + working_directory, + data_to_write, + ) # Perturb ################################################### # perform MCMC starting at each Smcap (total: N) for Nm_steps - scaled_proposal_covariance_matrix = (scale_factor_for_proposal_covariance ** 2) * weighted_sample_covariance_matrix # Proposal dist covariance matrix - - number_of_model_evaluations_in_this_stage = number_of_chains * number_of_MCMC_steps - write_stage_start_info_to_logfile(logfile, stage_number, beta, effective_sample_size, - scale_factor_for_proposal_covariance, log_evidence, number_of_model_evaluations_in_this_stage) + scaled_proposal_covariance_matrix = ( + scale_factor_for_proposal_covariance**2 + ) * weighted_sample_covariance_matrix # Proposal dist covariance matrix + + number_of_model_evaluations_in_this_stage = ( + number_of_chains * number_of_MCMC_steps + ) + write_stage_start_info_to_logfile( + logfile, + stage_number, + beta, + effective_sample_size, + scale_factor_for_proposal_covariance, + log_evidence, + number_of_model_evaluations_in_this_stage, + ) number_of_accepted_states_in_this_stage = 0 - iterables = [(sample_num, scaled_proposal_covariance_matrix, number_of_MCMC_steps, resampled_values[sample_num], - resampled_log_likelihood_values[sample_num], resampled_unnormalized_posterior_pdf_values[sample_num], beta, - number_of_accepted_states_in_this_stage, all_distributions_list, log_likelihood_function, model_parameters, - working_directory, default_rng(child_seeds[sample_num]), - calibration_data, number_of_experiments, covariance_matrix_list, - edp_names_list, edp_lengths_list, scale_factors, - shift_factors, driver_file, resampled_prediction_values[sample_num, :].reshape((1, -1))) - for sample_num in range(number_of_samples)] - + iterables = [ + ( + sample_num, + scaled_proposal_covariance_matrix, + number_of_MCMC_steps, + resampled_values[sample_num], + resampled_log_likelihood_values[sample_num], + resampled_unnormalized_posterior_pdf_values[sample_num], + beta, + number_of_accepted_states_in_this_stage, + all_distributions_list, + log_likelihood_function, + model_parameters, + working_directory, + default_rng(child_seeds[sample_num]), + calibration_data, + number_of_experiments, + covariance_matrix_list, + edp_names_list, + edp_lengths_list, + scale_factors, + shift_factors, + driver_file, + resampled_prediction_values[sample_num, :].reshape((1, -1)), + ) + for sample_num in range(number_of_samples) + ] + if run_type == "runningLocal": - write_eval_data_to_logfile(logfile, parallelize_MCMC, run_type, proc_count=processor_count, stage_num=stage_number) + write_eval_data_to_logfile( + logfile, + parallelize_MCMC, + run_type, + proc_count=processor_count, + stage_num=stage_number, + ) results = pool.starmap(tmcmcFunctions.MCMC_MH, iterables) else: - write_eval_data_to_logfile(logfile, parallelize_MCMC, run_type, MPI_size=MPI_size, stage_num=stage_number) + write_eval_data_to_logfile( + logfile, + parallelize_MCMC, + run_type, + MPI_size=MPI_size, + stage_num=stage_number, + ) results = list(executor.starmap(tmcmcFunctions.MCMC_MH, iterables)) - samples_list, loglikes_list, posterior_pdf_vals_list, num_accepts, all_proposals, all_PLP, preds_list = zip(*results) + ( + samples_list, + loglikes_list, + posterior_pdf_vals_list, + num_accepts, + all_proposals, + all_PLP, + preds_list, + ) = zip(*results) # for next beta sample_values = np.asarray(samples_list) log_likelihood_values = np.asarray(loglikes_list) unnormalized_posterior_pdf_values = np.asarray(posterior_pdf_vals_list) - prediction_values = np.asarray(preds_list).reshape((number_of_samples, -1)) + prediction_values = np.asarray(preds_list).reshape( + (number_of_samples, -1) + ) num_accepts = np.asarray(num_accepts) number_of_accepted_states_in_this_stage = sum(num_accepts) all_proposals = np.asarray(all_proposals) all_PLP = np.asarray(all_PLP) - total_number_of_model_evaluations += number_of_model_evaluations_in_this_stage - logfile.write("\n\n\t\tTotal number of model evaluations so far: {}".format(total_number_of_model_evaluations)) + total_number_of_model_evaluations += ( + number_of_model_evaluations_in_this_stage + ) + logfile.write( + "\n\n\t\tTotal number of model evaluations so far: {}".format( + total_number_of_model_evaluations + ) + ) # total observed acceptance rate - R = number_of_accepted_states_in_this_stage / number_of_model_evaluations_in_this_stage + R = ( + number_of_accepted_states_in_this_stage + / number_of_model_evaluations_in_this_stage + ) logfile.write(f"\n\n\t\tacceptance rate = {R:<9.6g}") - if adaptively_scale_proposal_covariance: # scale factor based on observed acceptance ratio + if ( + adaptively_scale_proposal_covariance + ): # scale factor based on observed acceptance ratio scale_factor_for_proposal_covariance = (1 / 9) + ((8 / 9) * R) - if adaptively_calculate_num_MCMC_steps: # Calculate Nm_steps based on observed acceptance rate + if ( + adaptively_calculate_num_MCMC_steps + ): # Calculate Nm_steps based on observed acceptance rate # increase max Nmcmc with stage number - number_of_MCMC_steps = min(number_of_MCMC_steps + 1, max_number_of_MCMC_steps) - logfile.write("\n\t\tadapted max MCMC steps = %d" % number_of_MCMC_steps) - - acc_rate = max(1. / number_of_model_evaluations_in_this_stage, R) - number_of_MCMC_steps = min(number_of_MCMC_steps, 1 + int(np.log(1 - 0.99) / np.log(1 - acc_rate))) + number_of_MCMC_steps = min( + number_of_MCMC_steps + 1, max_number_of_MCMC_steps + ) + logfile.write( + "\n\t\tadapted max MCMC steps = %d" % number_of_MCMC_steps + ) + + acc_rate = max(1.0 / number_of_model_evaluations_in_this_stage, R) + number_of_MCMC_steps = min( + number_of_MCMC_steps, + 1 + int(np.log(1 - 0.99) / np.log(1 - acc_rate)), + ) logfile.write("\n\t\tnext MCMC Nsteps = %d" % number_of_MCMC_steps) - logfile.write('\n\t\t==========================') + logfile.write("\n\t\t==========================") # save to trace - mytrace.append([sample_values, log_likelihood_values, np.ones(len(weights)), 'notValid', 1, 'notValid']) + mytrace.append( + [ + sample_values, + log_likelihood_values, + np.ones(len(weights)), + "notValid", + 1, + "notValid", + ] + ) # Write last stage data to '.csv' file data_to_write = np.hstack((sample_values, prediction_values)) - write_data_to_csvfile(logfile, total_number_of_models_in_ensemble, stage_number, model_number, - working_directory, data_to_write) - - write_data_to_tab_files(logfile, working_directory, model_number, model_parameters, - edp_names_list, edp_lengths_list, number_of_samples, dataToWrite=sample_values, - tab_file_name="dakotaTab.out", predictions=prediction_values) - - if parallelize_MCMC == 'yes': + write_data_to_csvfile( + logfile, + total_number_of_models_in_ensemble, + stage_number, + model_number, + working_directory, + data_to_write, + ) + + write_data_to_tab_files( + logfile, + working_directory, + model_number, + model_parameters, + edp_names_list, + edp_lengths_list, + number_of_samples, + dataToWrite=sample_values, + tab_file_name="dakotaTab.out", + predictions=prediction_values, + ) + + if parallelize_MCMC == "yes": if run_type == "runningLocal": pool.close() - logfile.write("\n\tClosed multiprocessing pool for runType: {}".format(run_type)) + logfile.write( + "\n\tClosed multiprocessing pool for runType: {}".format( + run_type + ) + ) else: executor.shutdown() - logfile.write("\n\tShutdown mpi4py executor pool for runType: {}".format(run_type)) + logfile.write( + "\n\tShutdown mpi4py executor pool for runType: {}".format( + run_type + ) + ) return mytrace, total_log_evidence diff --git a/modules/performUQ/UCSD_UQ/tmcmcFunctions.py b/modules/performUQ/UCSD_UQ/tmcmcFunctions.py index fb5c8c8b9..c14e30add 100644 --- a/modules/performUQ/UCSD_UQ/tmcmcFunctions.py +++ b/modules/performUQ/UCSD_UQ/tmcmcFunctions.py @@ -1,5 +1,5 @@ """ -authors: Mukesh Kumar Ramancha, Maitreya Manoj Kurumbhati, and Prof. J.P. Conte +authors: Mukesh Kumar Ramancha, Maitreya Manoj Kurumbhati, and Prof. J.P. Conte affiliation: University of California, San Diego """ @@ -175,7 +175,7 @@ def MCMC_MH_old( else: likelihood_proposal = -np.Inf # dont run the FE model posterior_proposal = -np.Inf - prediction_proposal = -np.Inf*np.ones_like(prediction_current) + prediction_proposal = -np.Inf * np.ones_like(prediction_current) log_acceptance = posterior_proposal - posterior_current all_proposals.append(proposal) @@ -268,7 +268,7 @@ def MCMC_MH( else: likelihood_proposal = -np.Inf # dont run the FE model posterior_proposal = -np.Inf - prediction_proposal = -np.Inf*np.ones_like(prediction_current) + prediction_proposal = -np.Inf * np.ones_like(prediction_current) log_acceptance = posterior_proposal - posterior_current all_proposals.append(proposal) @@ -355,9 +355,7 @@ def compute_beta_evidence(beta, log_likelihoods, logFile, threshold=1.0): max_beta = 1.0 dBeta = min(max_beta, 1.0 - beta) - weights, cov_weights, std_weights = get_weights( - dBeta, log_likelihoods - ) + weights, cov_weights, std_weights = get_weights(dBeta, log_likelihoods) while cov_weights > (threshold) or (std_weights == 0): dBeta = dBeta * 0.99 @@ -388,14 +386,14 @@ def compute_beta_evidence(beta, log_likelihoods, logFile, threshold=1.0): dBeta, log_likelihoods ) break - weights, cov_weights, std_weights = get_weights( - dBeta, log_likelihoods - ) + weights, cov_weights, std_weights = get_weights(dBeta, log_likelihoods) beta = beta + dBeta if beta > 0.95: beta = 1 - log_evidence = logsumexp(dBeta * log_likelihoods) - np.log(len(log_likelihoods)) + log_evidence = logsumexp(dBeta * log_likelihoods) - np.log( + len(log_likelihoods) + ) try: ESS = int(1 / np.sum((weights / np.sum(weights)) ** 2)) From 35a322606c03dfaf1461bea5bef042169f656d3b Mon Sep 17 00:00:00 2001 From: bsaakash <11618528+bsaakash@users.noreply.github.com> Date: Mon, 29 Jul 2024 15:29:01 -0700 Subject: [PATCH 02/26] abs - removing unused FEM data --- modules/performUQ/UCSD_UQ/parseData.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/performUQ/UCSD_UQ/parseData.py b/modules/performUQ/UCSD_UQ/parseData.py index d39b36e6b..0cd0f852f 100644 --- a/modules/performUQ/UCSD_UQ/parseData.py +++ b/modules/performUQ/UCSD_UQ/parseData.py @@ -38,7 +38,7 @@ def parseDataFunction(dakotaJsonFile, logFile, tmpSimCenterDir, mainscriptDir): applications = jsonInputs["Applications"] edpInputs = jsonInputs["EDP"] uqInputs = jsonInputs["UQ"] - femInputs = jsonInputs["FEM"] + # femInputs = jsonInputs["FEM"] rvInputs = jsonInputs["randomVariables"] # localAppDirInputs = jsonInputs['localAppDir'] # pythonInputs = jsonInputs['python'] From 8ea8059f7a4e19a0e7765e82e66cba4216b24728 Mon Sep 17 00:00:00 2001 From: bsaakash <11618528+bsaakash@users.noreply.github.com> Date: Mon, 29 Jul 2024 16:20:25 -0700 Subject: [PATCH 03/26] abs - commenting unused code --- modules/performUQ/UCSD_UQ/mainscript_tmcmc.py | 2 - modules/performUQ/UCSD_UQ/parseData.py | 126 +++++++++--------- 2 files changed, 62 insertions(+), 66 deletions(-) diff --git a/modules/performUQ/UCSD_UQ/mainscript_tmcmc.py b/modules/performUQ/UCSD_UQ/mainscript_tmcmc.py index c80a9e3f6..6c4239471 100644 --- a/modules/performUQ/UCSD_UQ/mainscript_tmcmc.py +++ b/modules/performUQ/UCSD_UQ/mainscript_tmcmc.py @@ -156,8 +156,6 @@ def main(input_args): number_of_samples, seed_value, calibration_data_filename, - loglikelihood_module, - write_outputs, variables_list, edp_names_list, edp_lengths_list, diff --git a/modules/performUQ/UCSD_UQ/parseData.py b/modules/performUQ/UCSD_UQ/parseData.py index 0cd0f852f..29b4bcb18 100644 --- a/modules/performUQ/UCSD_UQ/parseData.py +++ b/modules/performUQ/UCSD_UQ/parseData.py @@ -53,7 +53,7 @@ def parseDataFunction(dakotaJsonFile, logFile, tmpSimCenterDir, mainscriptDir): # numCol = spreadsheet['numCol'] # numRow = spreadsheet['numRow'] # summary = uqResultsInputs['summary'] - workingDir = jsonInputs["workingDir"] + # workingDir = jsonInputs["workingDir"] # Processing UQ inputs logFile.write("\n\t\tProcessing UQ inputs") @@ -71,66 +71,66 @@ def parseDataFunction(dakotaJsonFile, logFile, tmpSimCenterDir, mainscriptDir): if "parallelExecution" in uqInputs: parallelizeMCMC = uqInputs["parallelExecution"] - logFile.write("\n\t\t\tProcessing the log-likelihood script options") - # If log-likelihood script is provided, use that, otherwise, use default log-likelihood function - if ( - len(logLikelihoodFile) > 0 - ): # if the log-likelihood file is not an empty string - logFile.write( - "\n\t\t\t\tSearching for a user-defined log-likelihood script '{}'".format( - logLikelihoodFile - ) - ) - if os.path.exists(os.path.join(tmpSimCenterDir, logLikelihoodFile)): - logFile.write( - "\n\t\t\t\tFound log-likelihood file '{}' in {}.".format( - logLikelihoodFile, tmpSimCenterDir - ) - ) - logLikeModuleName = os.path.splitext(logLikelihoodFile)[0] - try: - import_module(logLikeModuleName) - except: - logFile.write( - "\n\t\t\t\tERROR: The log-likelihood script '{}' cannot be imported.".format( - os.path.join(tmpSimCenterDir, logLikelihoodFile) - ) - ) - raise - else: - logFile.write( - "\n\t\t\t\tERROR: The log-likelihood script '{}' cannot be found in {}.".format( - logLikelihoodFile, tmpSimCenterDir - ) - ) - raise FileNotFoundError( - "ERROR: The log-likelihood script '{}' cannot be found in {}.".format( - logLikelihoodFile, tmpSimCenterDir - ) - ) - else: - defaultLogLikeFileName = "defaultLogLikeScript.py" - defaultLogLikeDirectoryPath = mainscriptDir - sys.path.append(defaultLogLikeDirectoryPath) - logLikeModuleName = os.path.splitext(defaultLogLikeFileName)[0] - logFile.write("\n\t\t\t\tLog-likelihood script not provided.") - logFile.write( - "\n\t\t\t\tUsing the default log-likelihood script: \n\t\t\t\t\t{}".format( - os.path.join( - defaultLogLikeDirectoryPath, defaultLogLikeFileName - ) - ) - ) - try: - import_module(logLikeModuleName) - except: - logFile.write( - "\n\t\t\t\tERROR: The log-likelihood script '{}' cannot be imported.".format( - os.path.join(tmpSimCenterDir, logLikelihoodFile) - ) - ) - raise - logLikeModule = import_module(logLikeModuleName) + # logFile.write("\n\t\t\tProcessing the log-likelihood script options") + # # If log-likelihood script is provided, use that, otherwise, use default log-likelihood function + # if ( + # len(logLikelihoodFile) > 0 + # ): # if the log-likelihood file is not an empty string + # logFile.write( + # "\n\t\t\t\tSearching for a user-defined log-likelihood script '{}'".format( + # logLikelihoodFile + # ) + # ) + # if os.path.exists(os.path.join(tmpSimCenterDir, logLikelihoodFile)): + # logFile.write( + # "\n\t\t\t\tFound log-likelihood file '{}' in {}.".format( + # logLikelihoodFile, tmpSimCenterDir + # ) + # ) + # logLikeModuleName = os.path.splitext(logLikelihoodFile)[0] + # try: + # import_module(logLikeModuleName) + # except: + # logFile.write( + # "\n\t\t\t\tERROR: The log-likelihood script '{}' cannot be imported.".format( + # os.path.join(tmpSimCenterDir, logLikelihoodFile) + # ) + # ) + # raise + # else: + # logFile.write( + # "\n\t\t\t\tERROR: The log-likelihood script '{}' cannot be found in {}.".format( + # logLikelihoodFile, tmpSimCenterDir + # ) + # ) + # raise FileNotFoundError( + # "ERROR: The log-likelihood script '{}' cannot be found in {}.".format( + # logLikelihoodFile, tmpSimCenterDir + # ) + # ) + # else: + # defaultLogLikeFileName = "defaultLogLikeScript.py" + # defaultLogLikeDirectoryPath = mainscriptDir + # sys.path.append(defaultLogLikeDirectoryPath) + # logLikeModuleName = os.path.splitext(defaultLogLikeFileName)[0] + # logFile.write("\n\t\t\t\tLog-likelihood script not provided.") + # logFile.write( + # "\n\t\t\t\tUsing the default log-likelihood script: \n\t\t\t\t\t{}".format( + # os.path.join( + # defaultLogLikeDirectoryPath, defaultLogLikeFileName + # ) + # ) + # ) + # try: + # import_module(logLikeModuleName) + # except: + # logFile.write( + # "\n\t\t\t\tERROR: The log-likelihood script '{}' cannot be imported.".format( + # os.path.join(tmpSimCenterDir, logLikelihoodFile) + # ) + # ) + # raise + # logLikeModule = import_module(logLikeModuleName) # Processing EDP inputs logFile.write("\n\n\t\tProcessing EDP inputs") @@ -199,7 +199,7 @@ def parseDataFunction(dakotaJsonFile, logFile, tmpSimCenterDir, mainscriptDir): # else: # inputFileList.append(femInputs['inputFile']) # logFile.write('\n\t\t\t\tThe number of models is: {}'.format(nModels)) - writeFEMOutputs = True + # writeFEMOutputs = True # Variables variablesList = [] @@ -395,8 +395,6 @@ def parseDataFunction(dakotaJsonFile, logFile, tmpSimCenterDir, mainscriptDir): nSamples, seedValue, calDataFile, - logLikeModule, - writeFEMOutputs, variablesList, edpNamesList, edpLengthsList, From b5329afd2322db3798ea2de6fa1e0f5fc7657bb5 Mon Sep 17 00:00:00 2001 From: bsaakash <11618528+bsaakash@users.noreply.github.com> Date: Mon, 29 Jul 2024 16:39:22 -0700 Subject: [PATCH 04/26] abs - ignoring .coverage files --- .gitignore | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index a6472ca28..8ed2e0253 100644 --- a/.gitignore +++ b/.gitignore @@ -98,5 +98,4 @@ fmkTest.sh #Remove .vscode .vscode - - +.coverage \ No newline at end of file From fc486581eb48e057a2ef9594822169508339a3d4 Mon Sep 17 00:00:00 2001 From: bsaakash <11618528+bsaakash@users.noreply.github.com> Date: Thu, 8 Aug 2024 16:18:54 -0700 Subject: [PATCH 05/26] abs - fixed error message writing --- modules/performUQ/UCSD_UQ/UCSD_UQ.py | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/modules/performUQ/UCSD_UQ/UCSD_UQ.py b/modules/performUQ/UCSD_UQ/UCSD_UQ.py index 16c2c4327..183146552 100644 --- a/modules/performUQ/UCSD_UQ/UCSD_UQ.py +++ b/modules/performUQ/UCSD_UQ/UCSD_UQ.py @@ -1,11 +1,11 @@ import argparse import os import platform +import shlex import stat import subprocess -from pathlib import Path import sys -import shlex +from pathlib import Path def main(args): @@ -24,7 +24,6 @@ def main(args): runType = args.runType if runType in ["runningLocal"]: - if platform.system() == "Windows": pythonCommand = "python" driverFile = driverFile + ".bat" @@ -41,35 +40,23 @@ def main(args): st = os.stat(driverFile) os.chmod(driverFile, st.st_mode | stat.S_IEXEC) driverFile = "./" + driverFile - print("WORKFLOW: " + driverFile) command = ( f'"{pythonCommand}" "{mainScript}" "{tmpSimCenterDir}"' f' "{templateDir}" {runType} {driverFile} {workflowInput}' ) - print(command) command_list = shlex.split(command) - result = subprocess.run( - command_list, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - check=True, - text=True, - ) - err_file = Path(tmpSimCenterDir) / "UCSD_UQ.err" err_file.touch() try: + result = subprocess.run(command_list, capture_output=True, text=True) result.check_returncode() except subprocess.CalledProcessError: with open(err_file, "a") as f: - f.write(f"ERROR: {result.stderr}\n\n") - f.write(f"The command was: {result.args}\n\n") - f.write(f"The return code was: {result.returncode}\n\n") - f.write(f"The output of the command was: {result.stdout}\n\n") + f.write(f"ERROR: {result.stderr}") if __name__ == "__main__": From f2c90f2fcca1f66e2db2f9dcac54d9e841b01915 Mon Sep 17 00:00:00 2001 From: bsaakash <11618528+bsaakash@users.noreply.github.com> Date: Thu, 8 Aug 2024 16:45:59 -0700 Subject: [PATCH 06/26] abs - formatting files with bigger linewidth --- modules/performUQ/UCSD_UQ/UCSD_UQ.py | 1 + .../UCSD_UQ/calibration_utilities.py | 103 +++-------- .../performUQ/UCSD_UQ/defaultLogLikeScript.py | 28 +-- modules/performUQ/UCSD_UQ/mainscript.py | 7 +- .../mainscript_hierarchical_bayesian.py | 57 ++---- modules/performUQ/UCSD_UQ/mainscript_tmcmc.py | 59 ++---- modules/performUQ/UCSD_UQ/mwg_sampler.py | 172 ++++++------------ modules/performUQ/UCSD_UQ/parseData.py | 16 +- modules/performUQ/UCSD_UQ/pdfs.py | 8 +- modules/performUQ/UCSD_UQ/runFEM.py | 8 +- modules/performUQ/UCSD_UQ/runTMCMC.py | 79 +++----- modules/performUQ/UCSD_UQ/tmcmcFunctions.py | 36 +--- modules/performUQ/common/uq_utilities.py | 48 ++--- 13 files changed, 190 insertions(+), 432 deletions(-) diff --git a/modules/performUQ/UCSD_UQ/UCSD_UQ.py b/modules/performUQ/UCSD_UQ/UCSD_UQ.py index 183146552..0d636df46 100644 --- a/modules/performUQ/UCSD_UQ/UCSD_UQ.py +++ b/modules/performUQ/UCSD_UQ/UCSD_UQ.py @@ -45,6 +45,7 @@ def main(args): f'"{pythonCommand}" "{mainScript}" "{tmpSimCenterDir}"' f' "{templateDir}" {runType} {driverFile} {workflowInput}' ) + print(command) command_list = shlex.split(command) diff --git a/modules/performUQ/UCSD_UQ/calibration_utilities.py b/modules/performUQ/UCSD_UQ/calibration_utilities.py index 5b11b053b..f5553d07a 100644 --- a/modules/performUQ/UCSD_UQ/calibration_utilities.py +++ b/modules/performUQ/UCSD_UQ/calibration_utilities.py @@ -1,15 +1,14 @@ -import numpy as np -from scipy.linalg import block_diag import os import shutil +import sys import time -from numpy.typing import NDArray -from typing import Callable -from typing import TextIO from importlib import import_module -import sys +from typing import Callable, TextIO +import numpy as np import pdfs +from numpy.typing import NDArray +from scipy.linalg import block_diag class DataProcessingError(Exception): @@ -63,9 +62,7 @@ def getDefaultErrorVariances(self): # calibrated. There will be one such error variance value per response quantity. If there is only data from one # experiment,then the default error std.dev. value is assumed to be 5% of the absolute maximum value of the data # corresponding to that response quantity. - defaultErrorVariances = 1e-12 * np.ones_like( - self.edpLengthsList, dtype=float - ) + defaultErrorVariances = 1e-12 * np.ones_like(self.edpLengthsList, dtype=float) # defaultErrorVariances = np.zeros_like(self.edpLengthsList, dtype=float) if ( np.shape(self.calibrationData)[0] > 1 @@ -222,9 +219,7 @@ def createCovarianceMatrix(self): ) elif numCols == self.edpLengthsList[i]: covarianceTypeList.append("matrix") - logFile.write( - "\n\t\tA full covariance matrix provided." - ) + logFile.write("\n\t\tA full covariance matrix provided.") else: logFile.write( "\nERROR: The number of columns of data in the covariance matrix file {}" @@ -255,9 +250,7 @@ def createCovarianceMatrix(self): covarianceFile, self.edpLengthsList[i], numCols ) ) - logFile.write( - "\n\t\tCovariance matrix: {}".format(covMatrix) - ) + logFile.write("\n\t\tCovariance matrix: {}".format(covMatrix)) else: logFile.write( "\n\t\tDid not find a user supplied file. Using the default variance value." @@ -268,14 +261,10 @@ def createCovarianceMatrix(self): scalarVariance = np.array(self.defaultErrorVariances[i]) covarianceMatrixList.append(scalarVariance) covarianceTypeList.append("scalar") - logFile.write( - "\n\t\tCovariance matrix: {}".format(scalarVariance) - ) + logFile.write("\n\t\tCovariance matrix: {}".format(scalarVariance)) self.covarianceMatrixList = covarianceMatrixList self.covarianceTypeList = covarianceTypeList - logFile.write( - f"\n\nThe covariance matrix for prediction errors being used is:" - ) + logFile.write("\n\nThe covariance matrix for prediction errors being used is:") tmp = block_diag(*covarianceMatrixList) for row in tmp: rowString = " ".join([f"{col:14.8g}" for col in row]) @@ -341,9 +330,7 @@ def createTempCalDataFile(self, calDataFile): words = line.split() if len(words) == self.lineLength: self.numExperiments += 1 - tempLine = "{} {} ".format( - self.numExperiments, interface - ) + tempLine = "{} {} ".format(self.numExperiments, interface) for w in words: tempLine += "{} ".format(w) self.logFile.write( @@ -385,9 +372,7 @@ def readCleanedCalData(self): def getCalibrationData(self): calDataFile = os.path.join(self.workdirMain, self.calDataFileName) self.logFile.write( - "\nCalibration data file being processed: \n\t{}\n".format( - calDataFile - ) + "\nCalibration data file being processed: \n\t{}\n".format(calDataFile) ) self.createTempCalDataFile(calDataFile) self.readCleanedCalData() @@ -404,17 +389,13 @@ def transform_data_function( for j in range(len(list_of_data_segment_lengths)): slice_of_data = data_to_transform[ :, - currentPosition : currentPosition - + list_of_data_segment_lengths[j], + currentPosition : currentPosition + list_of_data_segment_lengths[j], ] slice_of_data = slice_of_data + list_of_shift_factors[j] data_to_transform[ :, - currentPosition : currentPosition - + list_of_data_segment_lengths[j], - ] = ( - slice_of_data / list_of_scale_factors[j] - ) + currentPosition : currentPosition + list_of_data_segment_lengths[j], + ] = slice_of_data / list_of_scale_factors[j] currentPosition += list_of_data_segment_lengths[j] return data_to_transform @@ -461,9 +442,7 @@ def computeScaleAndShiftFactors( currentPosition : currentPosition + self.edpLengthsList[j], ] absMax = np.absolute(np.max(calibrationDataSlice)) - if ( - absMax == 0 - ): # This is to handle the case if abs max of data = 0. + if absMax == 0: # This is to handle the case if abs max of data = 0. locShift = 1.0 absMax = 1.0 shiftFactors.append(locShift) @@ -484,9 +463,7 @@ def computeScaleAndShiftFactors( ] meanValue = np.nanmean(calibrationDataSlice) stdValue = np.nanstd(calibrationDataSlice) - if ( - stdValue == 0 - ): # This is to handle the case if stdev of data = 0. + if stdValue == 0: # This is to handle the case if stdev of data = 0. stdValue = 1.0 scaleFactors.append(stdValue) shiftFactors.append(-meanValue) @@ -523,11 +500,9 @@ def syncLogFile(logFile: TextIO): def make_distributions(variables): - all_distributions_list = [] for i in range(len(variables["names"])): - if variables["distributions"][i] == "Uniform": lower_limit = float(variables["Par1"][i]) upper_limit = float(variables["Par2"][i]) @@ -540,16 +515,12 @@ def make_distributions(variables): mean = float(variables["Par1"][i]) standard_deviation = float(variables["Par2"][i]) - all_distributions_list.append( - pdfs.Normal(mu=mean, sig=standard_deviation) - ) + all_distributions_list.append(pdfs.Normal(mu=mean, sig=standard_deviation)) if variables["distributions"][i] == "Half-Normal": standard_deviation = float(variables["Par1"][i]) - all_distributions_list.append( - pdfs.Halfnormal(sig=standard_deviation) - ) + all_distributions_list.append(pdfs.Halfnormal(sig=standard_deviation)) if variables["distributions"][i] == "Truncated-Normal": mean = float(variables["Par1"][i]) @@ -597,17 +568,13 @@ def make_distributions(variables): alpha = float(variables["Par1"][i]) beta = float(variables["Par2"][i]) - all_distributions_list.append( - pdfs.GumbelDist(alpha=alpha, beta=beta) - ) + all_distributions_list.append(pdfs.GumbelDist(alpha=alpha, beta=beta)) if variables["distributions"][i] == "Weibull": shape = float(variables["Par1"][i]) scale = float(variables["Par2"][i]) - all_distributions_list.append( - pdfs.WeibullDist(shape=shape, scale=scale) - ) + all_distributions_list.append(pdfs.WeibullDist(shape=shape, scale=scale)) if variables["distributions"][i] == "Exponential": lamda = float(variables["Par1"][i]) @@ -641,9 +608,7 @@ def make_distributions(variables): if variables["distributions"][i] == "Discrete": if variables["Par2"][i] is None: value = variables["Par1"][i] - all_distributions_list.append( - pdfs.ConstantInteger(value=value) - ) + all_distributions_list.append(pdfs.ConstantInteger(value=value)) else: values = float(variables["Par1"][i]) weights = float(variables["Par2"][i]) @@ -672,9 +637,7 @@ def __init__( self.list_of_scale_factors = list_of_scale_factors self.list_of_shift_factors = list_of_shift_factors self.workdir_main = workdir_main - self.full_path_to_tmcmc_code_directory = ( - full_path_to_tmcmc_code_directory - ) + self.full_path_to_tmcmc_code_directory = full_path_to_tmcmc_code_directory self.log_likelihood_file_name = log_likelihood_file_name sys.path.append(self.workdir_main) self._copy_log_likelihood_module() @@ -691,9 +654,7 @@ def _copy_log_likelihood_module(self): self.full_path_to_tmcmc_code_directory, self.log_likelihood_file_name, ) - dst = os.path.join( - self.workdir_main, self.log_likelihood_file_name - ) + dst = os.path.join(self.workdir_main, self.log_likelihood_file_name) try: shutil.copyfile(src, dst) except Exception: @@ -719,9 +680,7 @@ def _import_log_likelihood_module( return module # type: ignore def get_log_likelihood_function(self) -> Callable: - log_likelihood_module_name = os.path.splitext( - self.log_likelihood_file_name - )[0] + log_likelihood_module_name = os.path.splitext(self.log_likelihood_file_name)[0] module = self._import_log_likelihood_module(log_likelihood_module_name) return module.log_likelihood @@ -740,9 +699,7 @@ def _make_mean(self, response_num: int) -> NDArray: return np.zeros((self.list_of_data_segment_lengths[response_num])) def _make_covariance(self, response_num, cov_multiplier) -> NDArray: - return cov_multiplier * np.atleast_2d( - self.covariance_matrix_list[response_num] - ) + return cov_multiplier * np.atleast_2d(self.covariance_matrix_list[response_num]) def _make_input_for_log_likelihood_function(self, prediction) -> list: return [ @@ -761,13 +718,9 @@ def _loop_for_log_likelihood( currentPosition = 0 for j in range(self.num_response_quantities): length = self.list_of_data_segment_lengths[j] - residuals = allResiduals[ - i, currentPosition : currentPosition + length - ] + residuals = allResiduals[i, currentPosition : currentPosition + length] currentPosition = currentPosition + length - cov = self._make_covariance( - j, list_of_covariance_multipliers[j] - ) + cov = self._make_covariance(j, list_of_covariance_multipliers[j]) mean = self._make_mean(j) ll = self.log_likelihood_function(residuals, mean, cov) if not np.isnan(ll): diff --git a/modules/performUQ/UCSD_UQ/defaultLogLikeScript.py b/modules/performUQ/UCSD_UQ/defaultLogLikeScript.py index 37f43625c..7ffb63cba 100644 --- a/modules/performUQ/UCSD_UQ/defaultLogLikeScript.py +++ b/modules/performUQ/UCSD_UQ/defaultLogLikeScript.py @@ -71,34 +71,22 @@ def log_likelihood( if len(covarianceMatrixList) != numExperiments * numResponses: print( "ERROR: The expected number of covariance matrices is {}, but only {} were passed " - "in.".format( - numExperiments * numResponses, len(covarianceMatrixList) - ) + "in.".format(numExperiments * numResponses, len(covarianceMatrixList)) ) raise CovError( "ERROR: The expected number of covariance matrices is {}, but only {} were passed " - "in.".format( - numExperiments * numResponses, len(covarianceMatrixList) - ) + "in.".format(numExperiments * numResponses, len(covarianceMatrixList)) ) # Shift and normalize the prediction currentPosition = 0 for j in range(len(edpLengthsList)): - prediction[ - :, currentPosition : currentPosition + edpLengthsList[j] - ] = ( - prediction[ - :, currentPosition : currentPosition + edpLengthsList[j] - ] + prediction[:, currentPosition : currentPosition + edpLengthsList[j]] = ( + prediction[:, currentPosition : currentPosition + edpLengthsList[j]] + shiftFactors[j] ) - prediction[ - :, currentPosition : currentPosition + edpLengthsList[j] - ] = ( - prediction[ - :, currentPosition : currentPosition + edpLengthsList[j] - ] + prediction[:, currentPosition : currentPosition + edpLengthsList[j]] = ( + prediction[:, currentPosition : currentPosition + edpLengthsList[j]] / scaleFactors[j] ) currentPosition = currentPosition + edpLengthsList[j] @@ -114,9 +102,7 @@ def log_likelihood( for j in range(numResponses): # Get the residuals corresponding to this response variable length = edpLengthsList[j] - residuals = allResiduals[ - i, currentPosition : currentPosition + length - ] + residuals = allResiduals[i, currentPosition : currentPosition + length] currentPosition = currentPosition + length # Get the covariance matrix corresponding to this response variable diff --git a/modules/performUQ/UCSD_UQ/mainscript.py b/modules/performUQ/UCSD_UQ/mainscript.py index ab40498c6..9c65ff163 100644 --- a/modules/performUQ/UCSD_UQ/mainscript.py +++ b/modules/performUQ/UCSD_UQ/mainscript.py @@ -4,20 +4,19 @@ """ +import json +import shlex + # ====================================================================================================================== import sys -import json from pathlib import Path -import shlex path_to_common_uq = Path(__file__).parent.parent / "common" sys.path.append(str(path_to_common_uq)) -import uq_utilities # ====================================================================================================================== def main(input_args): - # # Initialize analysis # path_to_UCSD_UQ_directory = Path(input_args[2]).resolve().parent # path_to_working_directory = Path(input_args[3]).resolve() diff --git a/modules/performUQ/UCSD_UQ/mainscript_hierarchical_bayesian.py b/modules/performUQ/UCSD_UQ/mainscript_hierarchical_bayesian.py index a52f5223d..af2eaa355 100644 --- a/modules/performUQ/UCSD_UQ/mainscript_hierarchical_bayesian.py +++ b/modules/performUQ/UCSD_UQ/mainscript_hierarchical_bayesian.py @@ -3,15 +3,14 @@ from pathlib import Path import numpy as np +import preprocess_hierarchical_bayesian import scipy.linalg import scipy.stats -import preprocess_hierarchical_bayesian - path_to_common_uq = Path(__file__).parent.parent / "common" sys.path.append(str(path_to_common_uq)) -import uq_utilities import mwg_sampler +import uq_utilities def generate_initial_states( @@ -41,9 +40,9 @@ def generate_initial_states( np.array(state).reshape((num_rv, 1)) ) if "error_variances_scaled" in restart_data: - list_of_initial_states_of_error_variance_per_dataset = ( - restart_data["error_variances_scaled"] - ) + list_of_initial_states_of_error_variance_per_dataset = restart_data[ + "error_variances_scaled" + ] if "hyper_covariance" in restart_data: initial_state_of_hypercovariance = np.array( restart_data["hyper_covariance"] @@ -150,9 +149,7 @@ def main(input_args): list_of_proposal_covariance_kernels = [] list_of_cholesky_of_proposal_covariance_matrix = [] for dataset_number in range(num_datasets): - proposal_covariance_matrix = ( - proposal_scale_list[dataset_number] * cov_kernel - ) + proposal_covariance_matrix = proposal_scale_list[dataset_number] * cov_kernel list_of_proposal_covariance_kernels.append(cov_kernel) cholesky_of_proposal_covariance_matrix = scipy.linalg.cholesky( @@ -167,9 +164,7 @@ def main(input_args): list_of_prior_logpdf_values = [] iterable = [] for model_number in range(len(list_of_datasets)): - initial_state = list_of_initial_states_of_model_parameters[ - model_number - ] + initial_state = list_of_initial_states_of_model_parameters[model_number] x = transformation_function(initial_state) logpdf_of_initial_state = uq_utilities.multivariate_normal_logpdf( initial_state, @@ -195,47 +190,35 @@ def main(input_args): list_of_loglikelihood_at_initial_state = [] list_of_prior_logpdf_at_initial_state = [] for dataset_number, dataset in enumerate(list_of_datasets): - scaled_residual = ( - list_of_model_outputs[dataset_number] - dataset - ) / np.std(dataset) + scaled_residual = (list_of_model_outputs[dataset_number] - dataset) / np.std( + dataset + ) error_variance_sample_scaled = ( - list_of_initial_states_of_error_variance_per_dataset[ - dataset_number - ] + list_of_initial_states_of_error_variance_per_dataset[dataset_number] ) log_likelihood_at_initial_state = loglikelihood_function( scaled_residual, error_variance_sample_scaled, ) - prior_logpdf_at_initial_state = list_of_prior_logpdf_values[ - dataset_number - ] + prior_logpdf_at_initial_state = list_of_prior_logpdf_values[dataset_number] unnormalized_posterior_logpdf_at_initial_state = ( log_likelihood_at_initial_state + prior_logpdf_at_initial_state ) list_of_unnormalized_posterior_logpdf_at_initial_state.append( unnormalized_posterior_logpdf_at_initial_state ) - list_of_loglikelihood_at_initial_state.append( - log_likelihood_at_initial_state - ) - list_of_prior_logpdf_at_initial_state.append( - prior_logpdf_at_initial_state - ) + list_of_loglikelihood_at_initial_state.append(log_likelihood_at_initial_state) + list_of_prior_logpdf_at_initial_state.append(prior_logpdf_at_initial_state) results_directory_name = "sampling_results" results_directory_path = working_directory / results_directory_name results_directory_path.mkdir(parents=True, exist_ok=False) - tabular_results_file_base_name = ( - working_directory / "posterior_samples_table.out" - ) + tabular_results_file_base_name = working_directory / "posterior_samples_table.out" results_to_write = {} results_to_write["log_priors"] = list_of_prior_logpdf_at_initial_state - results_to_write["log_likelihoods"] = ( - list_of_loglikelihood_at_initial_state - ) + results_to_write["log_likelihoods"] = list_of_loglikelihood_at_initial_state results_to_write["unnormalized_log_posteriors"] = ( list_of_unnormalized_posterior_logpdf_at_initial_state ) @@ -250,9 +233,7 @@ def main(input_args): list_of_initial_states_of_error_variance_per_dataset ) - with open( - results_directory_path / f"sample_0.json", "w", encoding="utf-8" - ) as f: + with open(results_directory_path / "sample_0.json", "w", encoding="utf-8") as f: json.dump(results_to_write, f, indent=4) adaptivity_results = {} @@ -263,9 +244,7 @@ def main(input_args): cov_kernels_list = [] for cov_kernel in list_of_proposal_covariance_kernels: cov_kernels_list.append(cov_kernel.tolist()) - adaptivity_results["list_of_proposal_covariance_kernels"] = ( - cov_kernels_list - ) + adaptivity_results["list_of_proposal_covariance_kernels"] = cov_kernels_list with open( results_directory_path.parent / f"adaptivity_results_{0}.json", "w", diff --git a/modules/performUQ/UCSD_UQ/mainscript_tmcmc.py b/modules/performUQ/UCSD_UQ/mainscript_tmcmc.py index 6c4239471..53e340ab4 100644 --- a/modules/performUQ/UCSD_UQ/mainscript_tmcmc.py +++ b/modules/performUQ/UCSD_UQ/mainscript_tmcmc.py @@ -9,26 +9,24 @@ import sys import time from typing import TextIO -import numpy as np -from parseData import parseDataFunction -from runTMCMC import run_TMCMC +import numpy as np from calibration_utilities import ( - CovarianceMatrixPreparer, CalDataPreparer, + CovarianceMatrixPreparer, DataTransformer, + LogLikelihoodHandler, createLogFile, - syncLogFile, make_distributions, - LogLikelihoodHandler, + syncLogFile, ) +from parseData import parseDataFunction +from runTMCMC import run_TMCMC # ====================================================================================================================== -def computeModelPosteriorProbabilities( - modelPriorProbabilities, modelEvidences -): +def computeModelPosteriorProbabilities(modelPriorProbabilities, modelEvidences): denominator = np.dot(modelPriorProbabilities, modelEvidences) return modelPriorProbabilities * modelEvidences / denominator @@ -148,9 +146,7 @@ def main(input_args): input_json_filename_full_path = input_json_filename logfile.write("\n\n==========================") logfile.write( - "\nParsing the json input file {}".format( - input_json_filename_full_path - ) + "\nParsing the json input file {}".format(input_json_filename_full_path) ) ( number_of_samples, @@ -243,12 +239,8 @@ def main(input_args): logfile, run_type, ) - defaultErrorVariances = ( - cov_matrix_options_instance.getDefaultErrorVariances() - ) - covariance_matrix_list = ( - cov_matrix_options_instance.createCovarianceMatrix() - ) + defaultErrorVariances = cov_matrix_options_instance.getDefaultErrorVariances() + covariance_matrix_list = cov_matrix_options_instance.createCovarianceMatrix() # ====================================================================================================================== # Get log-likelihood function @@ -278,14 +270,11 @@ def main(input_args): # number of max MCMC steps number_of_MCMC_steps = ( - tmcmc_data_instance.numBurnInSteps - + tmcmc_data_instance.numStepsAfterBurnIn + tmcmc_data_instance.numBurnInSteps + tmcmc_data_instance.numStepsAfterBurnIn ) max_number_of_MCMC_steps = 10 logfile.write( - "\n\tNumber of MCMC steps in first stage: {}".format( - number_of_MCMC_steps - ) + "\n\tNumber of MCMC steps in first stage: {}".format(number_of_MCMC_steps) ) logfile.write( "\n\tMax. number of MCMC steps in any stage: {}".format( @@ -297,9 +286,7 @@ def main(input_args): # ====================================================================================================================== # Initialize variables to store prior model probability and evidence - model_prior_probabilities = np.ones((len(variables_list),)) / len( - variables_list - ) + model_prior_probabilities = np.ones((len(variables_list),)) / len(variables_list) model_evidences = np.ones_like(model_prior_probabilities) logfile.write("\n\n==========================") @@ -307,18 +294,12 @@ def main(input_args): # For each model: for model_number, parameters_of_model in enumerate(variables_list): logfile.write("\n\n\t==========================") - logfile.write( - "\n\tStarting analysis for model {}".format(model_number + 1) - ) + logfile.write("\n\tStarting analysis for model {}".format(model_number + 1)) logfile.write("\n\t==========================") # Assign probability distributions to the parameters of the model - logfile.write( - "\n\t\tAssigning probability distributions to the parameters" - ) - all_distributions_list = make_distributions( - variables=parameters_of_model - ) + logfile.write("\n\t\tAssigning probability distributions to the parameters") + all_distributions_list = make_distributions(variables=parameters_of_model) # Run the Algorithm logfile.write("\n\n\t==========================") @@ -397,9 +378,7 @@ def main(input_args): model_evidences[model_number] = evidence logfile.write("\n\n\t==========================") - logfile.write( - "\n\tCompleted analysis for model {}".format(model_number + 1) - ) + logfile.write("\n\tCompleted analysis for model {}".format(model_number + 1)) logfile.write("\n\t==========================") syncLogFile(logfile) @@ -420,9 +399,7 @@ def main(input_args): # ====================================================================================================================== logfile.write("\nUCSD_UQ engine workflow complete!\n") - logfile.write( - "\nTime taken: {:0.2f} minutes\n\n".format((time.time() - t1) / 60) - ) + logfile.write("\nTime taken: {:0.2f} minutes\n\n".format((time.time() - t1) / 60)) syncLogFile(logfile) diff --git a/modules/performUQ/UCSD_UQ/mwg_sampler.py b/modules/performUQ/UCSD_UQ/mwg_sampler.py index a3e7dff3b..ae43c68a9 100644 --- a/modules/performUQ/UCSD_UQ/mwg_sampler.py +++ b/modules/performUQ/UCSD_UQ/mwg_sampler.py @@ -1,9 +1,9 @@ import json +from pathlib import Path + import numpy as np import scipy -from pathlib import Path - path_to_common_uq = Path(__file__).parent.parent / "common" import sys @@ -87,16 +87,14 @@ def _draw_one_sample( uq_utilities.get_standard_normal_random_variates(prngs_rvs) ) cholesky_decomposition_of_proposal_covariance_matrix = ( - list_of_cholesky_decomposition_of_proposal_covariance_matrix[ - dataset_number - ] + list_of_cholesky_decomposition_of_proposal_covariance_matrix[dataset_number] ) move = cholesky_decomposition_of_proposal_covariance_matrix @ np.array( standard_normal_random_variates ).reshape((-1, 1)) - current_state = np.array( - list_of_current_states[dataset_number] - ).reshape((-1, 1)) + current_state = np.array(list_of_current_states[dataset_number]).reshape( + (-1, 1) + ) proposed_state = current_state + move x = transformation_function(proposed_state) model_iterable = [0, x] @@ -126,22 +124,18 @@ def _draw_one_sample( for dataset_number, dataset in enumerate(list_of_datasets): proposed_state = list_of_proposed_states[dataset_number] - prior_logpdf_at_proposed_state = ( - uq_utilities.multivariate_normal_logpdf( - proposed_state, - current_mean_sample, - current_cov_sample, - ) - ) - list_of_logpdf_of_proposed_states.append( - prior_logpdf_at_proposed_state + prior_logpdf_at_proposed_state = uq_utilities.multivariate_normal_logpdf( + proposed_state, + current_mean_sample, + current_cov_sample, ) - scaled_residual = ( - list_of_model_outputs[dataset_number] - dataset - ) / np.std(dataset) - error_variance_sample_scaled = ( - list_of_current_error_variance_samples_scaled[dataset_number] + list_of_logpdf_of_proposed_states.append(prior_logpdf_at_proposed_state) + scaled_residual = (list_of_model_outputs[dataset_number] - dataset) / np.std( + dataset ) + error_variance_sample_scaled = list_of_current_error_variance_samples_scaled[ + dataset_number + ] log_likelihood_at_proposed_state = loglikelihood_function( scaled_residual, error_variance_sample_scaled, @@ -150,9 +144,7 @@ def _draw_one_sample( log_likelihood_at_proposed_state + prior_logpdf_at_proposed_state ) unnormalized_posterior_logpdf_at_current_state = ( - list_of_unnormalized_posterior_logpdf_at_current_state[ - dataset_number - ] + list_of_unnormalized_posterior_logpdf_at_current_state[dataset_number] ) log_hastings_ratio = ( unnormalized_posterior_logpdf_at_proposed_state @@ -160,9 +152,7 @@ def _draw_one_sample( ) list_of_log_hastings_ratios.append(log_hastings_ratio) list_of_log_likelihoods.append(log_likelihood_at_proposed_state) - standard_uniform_random_variate = prngs_algorithm[ - dataset_number - ].uniform() + standard_uniform_random_variate = prngs_algorithm[dataset_number].uniform() proposed_state = list_of_proposed_states[dataset_number] current_state = list_of_current_states[dataset_number] if (log_hastings_ratio >= 0) | ( @@ -174,9 +164,7 @@ def _draw_one_sample( ) list_of_log_likes.append(log_likelihood_at_proposed_state) list_of_log_priors.append(prior_logpdf_at_proposed_state) - num_accepts_list[dataset_number] = ( - num_accepts_list[dataset_number] + 1 - ) + num_accepts_list[dataset_number] = num_accepts_list[dataset_number] + 1 else: new_state = current_state list_of_log_posteriors.append( @@ -190,9 +178,7 @@ def _draw_one_sample( ) new_state = np.array(new_state).reshape((-1, 1)) list_of_new_states.append(new_state) - list_of_parameter_samples.append( - transformation_function(new_state).tolist() - ) + list_of_parameter_samples.append(transformation_function(new_state).tolist()) sse = scaled_residual @ scaled_residual list_of_sse.append(sse) @@ -207,12 +193,8 @@ def _draw_one_sample( prngs_edps[dataset_number], alpha_n, beta_n ) ).item() - list_of_error_variance_samples_scaled.append( - new_error_variance_sample_scaled - ) - new_error_variance_sample = ( - np.var(dataset) * new_error_variance_sample_scaled - ) + list_of_error_variance_samples_scaled.append(new_error_variance_sample_scaled) + new_error_variance_sample = np.var(dataset) * new_error_variance_sample_scaled list_of_error_variance_samples.append(new_error_variance_sample) n = num_datasets @@ -248,9 +230,7 @@ def _draw_one_sample( one_sample = {} one_sample["new_states"] = list_of_new_states - one_sample["error_variances_scaled"] = ( - list_of_error_variance_samples_scaled - ) + one_sample["error_variances_scaled"] = list_of_error_variance_samples_scaled one_sample["hyper_covariance"] = covariance_sample one_sample["hyper_mean"] = mean_sample @@ -265,14 +245,12 @@ def _draw_one_sample( else: new_states_list.append(item.tolist()) results_to_write["new_states"] = new_states_list - results_to_write["error_variances_scaled"] = ( - list_of_error_variance_samples_scaled - ) + results_to_write["error_variances_scaled"] = list_of_error_variance_samples_scaled results_to_write["hyper_covariance"] = covariance_sample.tolist() results_to_write["hyper_mean"] = mean_sample.tolist() - results_to_write[ - "updated_parameters_of_normal_inverse_wishart_distribution" - ] = updated_parameters + results_to_write["updated_parameters_of_normal_inverse_wishart_distribution"] = ( + updated_parameters + ) for dataset_number in range(num_datasets): x = list_of_parameter_samples[dataset_number] @@ -300,9 +278,7 @@ def _draw_one_sample( tabular_results_file_name, string_to_write ) - with open( - results_directory_path / f"sample_{sample_number+1}.json", "w" - ) as f: + with open(results_directory_path / f"sample_{sample_number+1}.json", "w") as f: json.dump(results_to_write, f, indent=4) return one_sample, results_to_write @@ -398,15 +374,11 @@ def metropolis_within_gibbs_sampler( if "Tuning Period" in uq_inputs: tuning_period = uq_inputs["Tuning Period"] num_samples = uq_inputs["Sample Size"] + tuning_period - parent_distribution_prng = ( - uq_utilities.get_list_of_pseudo_random_number_generators( - 10 * random_state, 1 - )[0] - ) + parent_distribution_prng = uq_utilities.get_list_of_pseudo_random_number_generators( + 10 * random_state, 1 + )[0] - initial_list_of_proposal_covariance_kernels = ( - list_of_proposal_covariance_kernels - ) + initial_list_of_proposal_covariance_kernels = list_of_proposal_covariance_kernels for dataset_number in range(num_datasets): tabular_results_file_name = ( @@ -446,18 +418,14 @@ def metropolis_within_gibbs_sampler( for rv in rv_inputs: rv_mean_string_list.append(f'mean_{rv["name"]}') rv_names_list.append(rv["name"]) - list_of_hyperparameter_header_strings.append( - "\t".join(rv_mean_string_list) - ) + list_of_hyperparameter_header_strings.append("\t".join(rv_mean_string_list)) rv_covariance_string_list = [] for i in range(len(rv_names_list)): for j in range(i, len(rv_names_list)): rv_covariance_string_list.append( f"cov_{rv_names_list[i]}_{rv_names_list[j]}" ) - list_of_hyperparameter_header_strings.append( - "\t".join(rv_covariance_string_list) - ) + list_of_hyperparameter_header_strings.append("\t".join(rv_covariance_string_list)) hyperparameter_header_string = ( "\t".join(list_of_hyperparameter_header_strings) + "\n" ) @@ -544,13 +512,9 @@ def metropolis_within_gibbs_sampler( proposal_scale = proposal_scale_list[dataset_number] proposal_scale = tune(proposal_scale, acc_rate) proposal_scale_list[dataset_number] = proposal_scale - cov_kernel = list_of_proposal_covariance_kernels[ - dataset_number - ] + cov_kernel = list_of_proposal_covariance_kernels[dataset_number] if num_accepts > num_rv: - states = get_states_from_samples_list( - samples, dataset_number - ) + states = get_states_from_samples_list(samples, dataset_number) samples_array = np.array(states[-tuning_interval:]).T try: cov_kernel = np.cov(samples_array) @@ -560,15 +524,11 @@ def metropolis_within_gibbs_sampler( f" {dataset_number}, Exception in covariance" f" calculation: {exc}" ) - cov_kernel = list_of_proposal_covariance_kernels[ - dataset_number - ] + cov_kernel = list_of_proposal_covariance_kernels[dataset_number] proposal_covariance_matrix = cov_kernel * proposal_scale try: - cholesky_of_proposal_covariance_matrix = ( - scipy.linalg.cholesky( - proposal_covariance_matrix, lower=True - ) + cholesky_of_proposal_covariance_matrix = scipy.linalg.cholesky( + proposal_covariance_matrix, lower=True ) except Exception as exc: print( @@ -576,34 +536,24 @@ def metropolis_within_gibbs_sampler( f" {dataset_number}, Exception in cholesky" f" calculation: {exc}" ) - cov_kernel = list_of_proposal_covariance_kernels[ - dataset_number - ] + cov_kernel = list_of_proposal_covariance_kernels[dataset_number] proposal_covariance_matrix = cov_kernel * proposal_scale - cholesky_of_proposal_covariance_matrix = ( - scipy.linalg.cholesky( - proposal_covariance_matrix, lower=True - ) + cholesky_of_proposal_covariance_matrix = scipy.linalg.cholesky( + proposal_covariance_matrix, lower=True ) - list_of_cholesky_of_proposal_covariance_matrix[ - dataset_number - ] = cholesky_of_proposal_covariance_matrix - list_of_proposal_covariance_kernels[dataset_number] = ( - cov_kernel + list_of_cholesky_of_proposal_covariance_matrix[dataset_number] = ( + cholesky_of_proposal_covariance_matrix ) + list_of_proposal_covariance_kernels[dataset_number] = cov_kernel num_accepts_list = [0] * num_datasets adaptivity_results = {} - adaptivity_results["list_of_acceptance_rates"] = ( - list_of_acceptance_rates - ) + adaptivity_results["list_of_acceptance_rates"] = list_of_acceptance_rates adaptivity_results["proposal_scale_list"] = proposal_scale_list cov_kernels_list = [] for cov_kernel in list_of_proposal_covariance_kernels: cov_kernels_list.append(cov_kernel.tolist()) - adaptivity_results["list_of_proposal_covariance_kernels"] = ( - cov_kernels_list - ) + adaptivity_results["list_of_proposal_covariance_kernels"] = cov_kernels_list with open( results_directory_path.parent / f"adaptivity_results_{sample_number}.json", @@ -619,15 +569,11 @@ def metropolis_within_gibbs_sampler( hyper_covariance = current_covariance_sample for i in range(len(rv_names_list)): for j in range(i, len(rv_names_list)): - hyper_covariance_string_list.append( - f"{hyper_covariance[i][j]}" - ) + hyper_covariance_string_list.append(f"{hyper_covariance[i][j]}") list_of_hyperparameter_value_strings = [] list_of_hyperparameter_value_strings.append(f"{sample_number+1}") list_of_hyperparameter_value_strings.append("0") - list_of_hyperparameter_value_strings.append( - "\t".join(hyper_mean_string_list) - ) + list_of_hyperparameter_value_strings.append("\t".join(hyper_mean_string_list)) list_of_hyperparameter_value_strings.append( "\t".join(hyper_covariance_string_list) ) @@ -648,8 +594,7 @@ def metropolis_within_gibbs_sampler( 1 / 5 * num_samples ) for i in range( - num_samples - - n_samples_for_mean_of_updated_predictive_distribution_parameters, + num_samples - n_samples_for_mean_of_updated_predictive_distribution_parameters, num_samples, ): with open(results_directory_path / f"sample_{i+1}.json", "r") as f: @@ -670,9 +615,7 @@ def metropolis_within_gibbs_sampler( df = nu_n_mean - num_datasets + 1 loc = mu_n_mean shape = (lambda_n_mean + 1) / (lambda_n_mean * df) * psi_n_mean - predictive_distribution = scipy.stats.multivariate_t( - loc=loc, shape=shape, df=df - ) + predictive_distribution = scipy.stats.multivariate_t(loc=loc, shape=shape, df=df) for sample_number in range(num_samples): sample_from_predictive_t_distribution = predictive_distribution.rvs( random_state=parent_distribution_prng @@ -680,13 +623,9 @@ def metropolis_within_gibbs_sampler( sample_from_predictive_distribution = transformation_function( sample_from_predictive_t_distribution ) - while ( - np.sum(np.isfinite(sample_from_predictive_distribution)) < num_rv - ): - sample_from_predictive_t_distribution = ( - predictive_distribution.rvs( - random_state=parent_distribution_prng - ) + while np.sum(np.isfinite(sample_from_predictive_distribution)) < num_rv: + sample_from_predictive_t_distribution = predictive_distribution.rvs( + random_state=parent_distribution_prng ) sample_from_predictive_distribution = transformation_function( sample_from_predictive_t_distribution @@ -703,8 +642,7 @@ def metropolis_within_gibbs_sampler( "\t".join(predictive_distribution_sample_values_list) ) predictive_distribution_sample_value_string = ( - "\t".join(list_of_predictive_distribution_sample_value_strings) - + "\n" + "\t".join(list_of_predictive_distribution_sample_value_strings) + "\n" ) uq_utilities._write_to_tabular_results_file( tabular_results_file_base_name, diff --git a/modules/performUQ/UCSD_UQ/parseData.py b/modules/performUQ/UCSD_UQ/parseData.py index 29b4bcb18..d59aac9cc 100644 --- a/modules/performUQ/UCSD_UQ/parseData.py +++ b/modules/performUQ/UCSD_UQ/parseData.py @@ -4,15 +4,9 @@ """ +import itertools import json import os -import sys -import time -from importlib import import_module -from shutil import copyfile - -import numpy as np -import itertools class DataProcessingError(Exception): @@ -219,9 +213,7 @@ def parseDataFunction(dakotaJsonFile, logFile, tmpSimCenterDir, mainscriptDir): for ind in range(nModels): logFile.write("\n\t\t\t\tModel number: {}".format(ind)) # Processing RV inputs - logFile.write( - "\n\t\t\t\t\tCreating priors for model number {}".format(ind) - ) + logFile.write("\n\t\t\t\t\tCreating priors for model number {}".format(ind)) logFile.write("\n\t\t\t\t\t\tProcessing RV inputs") for i, rv in enumerate(rvInputs): variablesList[ind]["names"].append(rv["name"]) @@ -287,9 +279,7 @@ def parseDataFunction(dakotaJsonFile, logFile, tmpSimCenterDir, mainscriptDir): variablesList[ind]["Par2"].append(rv["betaparam"]) variablesList[ind]["Par3"].append(None) variablesList[ind]["Par4"].append(None) - paramString = "params: {}, {}".format( - rv["alphaparam"], rv["betaparam"] - ) + paramString = "params: {}, {}".format(rv["alphaparam"], rv["betaparam"]) elif rv["distribution"] == "Weibull": variablesList[ind]["Par1"].append(rv["shapeparam"]) variablesList[ind]["Par2"].append(rv["scaleparam"]) diff --git a/modules/performUQ/UCSD_UQ/pdfs.py b/modules/performUQ/UCSD_UQ/pdfs.py index abfc067fd..4114f1528 100644 --- a/modules/performUQ/UCSD_UQ/pdfs.py +++ b/modules/performUQ/UCSD_UQ/pdfs.py @@ -141,9 +141,7 @@ def __init__(self, alpha, beta, lowerbound, upperbound): self.beta = beta self.lowerbound = lowerbound self.upperbound = upperbound - self.dist = stats.beta( - self.alpha, self.beta, self.lowerbound, self.upperbound - ) + self.dist = stats.beta(self.alpha, self.beta, self.lowerbound, self.upperbound) def generate_rns(self, N): return self.dist.rvs(size=N) @@ -257,9 +255,7 @@ def __init__(self, values, weights): self.values = values self.weights = weights self.probabilities = self.weights / np.sum(self.weights) - self.log_probabilities = np.log(self.weights) - np.log( - np.sum(self.weights) - ) + self.log_probabilities = np.log(self.weights) - np.log(np.sum(self.weights)) self.rng = np.random.default_rng() def generate_rns(self, N): diff --git a/modules/performUQ/UCSD_UQ/runFEM.py b/modules/performUQ/UCSD_UQ/runFEM.py index bb3f23f22..856bab89d 100644 --- a/modules/performUQ/UCSD_UQ/runFEM.py +++ b/modules/performUQ/UCSD_UQ/runFEM.py @@ -6,8 +6,9 @@ """ import os -import subprocess import shutil +import subprocess + import numpy as np @@ -27,9 +28,7 @@ def copytree(src, dst, symlinks=False, ignore=None): ): shutil.copy2(s, d) except Exception as ex: - msg = ( - f"Could not copy {s}. The following error occurred: \n{ex}" - ) + msg = f"Could not copy {s}. The following error occurred: \n{ex}" return msg return "0" @@ -106,3 +105,4 @@ def runFEM( ll = -np.inf return (ll, preds) + return (ll, preds) diff --git a/modules/performUQ/UCSD_UQ/runTMCMC.py b/modules/performUQ/UCSD_UQ/runTMCMC.py index 92f8021ce..9fb8ad4ae 100644 --- a/modules/performUQ/UCSD_UQ/runTMCMC.py +++ b/modules/performUQ/UCSD_UQ/runTMCMC.py @@ -4,14 +4,15 @@ modified: Aakash Bangalore Satish, NHERI SimCenter, UC Berkeley """ -import numpy as np -import tmcmcFunctions +import csv import multiprocessing as mp +import os from multiprocessing import Pool -from runFEM import runFEM + +import numpy as np +import tmcmcFunctions from numpy.random import SeedSequence, default_rng -import os -import csv +from runFEM import runFEM def write_stage_start_info_to_logfile( @@ -58,9 +59,7 @@ def write_eval_data_to_logfile( else: logfile.write("\n\n\t\tLocal run - MCMC steps") logfile.write( - "\n\t\t\tNumber of processors being used: {}".format( - proc_count - ) + "\n\t\t\tNumber of processors being used: {}".format(proc_count) ) else: if stage_num == 0: @@ -109,9 +108,7 @@ def create_headings( def get_prediction_from_workdirs(i, working_directory): workdir_string = "workdir." + str(i + 1) prediction = np.atleast_2d( - np.genfromtxt( - os.path.join(working_directory, workdir_string, "results.out") - ) + np.genfromtxt(os.path.join(working_directory, workdir_string, "results.out")) ).reshape((1, -1)) return prediction @@ -128,7 +125,6 @@ def write_data_to_tab_files( tab_file_name, predictions, ): - tab_file_full_path = os.path.join(working_directory, tab_file_name) write_outputs = True headings = create_headings( @@ -145,9 +141,7 @@ def write_data_to_tab_files( if model_number == 0: f.write(headings) for i in range(number_of_samples): - row_string = ( - f"{i + 1 + number_of_samples*model_number}\t{model_number+1}\t" - ) + row_string = f"{i + 1 + number_of_samples*model_number}\t{model_number+1}\t" for j in range(len(model_parameters["names"])): row_string += f"{dataToWrite[i, j]}\t" if write_outputs: # write the output data @@ -171,19 +165,13 @@ def write_data_to_csvfile( data_to_write, ): logfile.write( - "\n\n\t\tWriting samples from stage {} to csv file".format( - stage_number - 1 - ) + "\n\n\t\tWriting samples from stage {} to csv file".format(stage_number - 1) ) if total_number_of_models_in_ensemble > 1: - string_to_append = ( - f"resultsStage{stage_number - 1}_Model_{model_number+1}.csv" - ) + string_to_append = f"resultsStage{stage_number - 1}_Model_{model_number+1}.csv" else: string_to_append = f"resultsStage{stage_number - 1}.csv" - resultsFilePath = os.path.join( - os.path.abspath(working_directory), string_to_append - ) + resultsFilePath = os.path.join(os.path.abspath(working_directory), string_to_append) with open(resultsFilePath, "w", newline="") as csvfile: csvWriter = csv.writer(csvfile) @@ -249,14 +237,9 @@ def run_TMCMC( # Evaluate posterior at Sm prior_pdf_values = np.array( - [ - tmcmcFunctions.log_prior(s, all_distributions_list) - for s in sample_values - ] + [tmcmcFunctions.log_prior(s, all_distributions_list) for s in sample_values] ).squeeze() - unnormalized_posterior_pdf_values = ( - prior_pdf_values # prior = post for beta = 0 - ) + unnormalized_posterior_pdf_values = prior_pdf_values # prior = post for beta = 0 iterables = [ ( @@ -312,9 +295,7 @@ def run_TMCMC( log_likelihoods_list.append(output[0]) predictions_list.append(output[1]) log_likelihood_values = np.array(log_likelihoods_list).squeeze() - prediction_values = np.array(predictions_list).reshape( - (number_of_samples, -1) - ) + prediction_values = np.array(predictions_list).reshape((number_of_samples, -1)) total_number_of_model_evaluations = number_of_samples logfile.write( @@ -380,12 +361,10 @@ def run_TMCMC( resampled_values = sample_values[resample_ids] resampled_log_likelihood_values = log_likelihood_values[resample_ids] - resampled_unnormalized_posterior_pdf_values = ( - unnormalized_posterior_pdf_values[resample_ids] - ) - resampled_prediction_values = np.atleast_2d( - prediction_values[resample_ids, :] - ) + resampled_unnormalized_posterior_pdf_values = unnormalized_posterior_pdf_values[ + resample_ids + ] + resampled_prediction_values = np.atleast_2d(prediction_values[resample_ids, :]) # save to trace # stage m: samples, likelihood, weights, next stage ESS, next stage beta, resampled samples @@ -491,18 +470,14 @@ def run_TMCMC( sample_values = np.asarray(samples_list) log_likelihood_values = np.asarray(loglikes_list) unnormalized_posterior_pdf_values = np.asarray(posterior_pdf_vals_list) - prediction_values = np.asarray(preds_list).reshape( - (number_of_samples, -1) - ) + prediction_values = np.asarray(preds_list).reshape((number_of_samples, -1)) num_accepts = np.asarray(num_accepts) number_of_accepted_states_in_this_stage = sum(num_accepts) all_proposals = np.asarray(all_proposals) all_PLP = np.asarray(all_PLP) - total_number_of_model_evaluations += ( - number_of_model_evaluations_in_this_stage - ) + total_number_of_model_evaluations += number_of_model_evaluations_in_this_stage logfile.write( "\n\n\t\tTotal number of model evaluations so far: {}".format( total_number_of_model_evaluations @@ -527,9 +502,7 @@ def run_TMCMC( number_of_MCMC_steps = min( number_of_MCMC_steps + 1, max_number_of_MCMC_steps ) - logfile.write( - "\n\t\tadapted max MCMC steps = %d" % number_of_MCMC_steps - ) + logfile.write("\n\t\tadapted max MCMC steps = %d" % number_of_MCMC_steps) acc_rate = max(1.0 / number_of_model_evaluations_in_this_stage, R) number_of_MCMC_steps = min( @@ -580,16 +553,12 @@ def run_TMCMC( if run_type == "runningLocal": pool.close() logfile.write( - "\n\tClosed multiprocessing pool for runType: {}".format( - run_type - ) + "\n\tClosed multiprocessing pool for runType: {}".format(run_type) ) else: executor.shutdown() logfile.write( - "\n\tShutdown mpi4py executor pool for runType: {}".format( - run_type - ) + "\n\tShutdown mpi4py executor pool for runType: {}".format(run_type) ) return mytrace, total_log_evidence diff --git a/modules/performUQ/UCSD_UQ/tmcmcFunctions.py b/modules/performUQ/UCSD_UQ/tmcmcFunctions.py index c14e30add..d535166a1 100644 --- a/modules/performUQ/UCSD_UQ/tmcmcFunctions.py +++ b/modules/performUQ/UCSD_UQ/tmcmcFunctions.py @@ -61,9 +61,7 @@ def compute_beta(beta, likelihoods, prev_ESS, threshold): return new_beta, Wm, ESS -def compute_beta_evidence_old( - beta, log_likelihoods, log_evidence, prev_ESS, threshold -): +def compute_beta_evidence_old(beta, log_likelihoods, log_evidence, prev_ESS, threshold): old_beta = beta min_beta = beta max_beta = 2.0 @@ -149,9 +147,7 @@ def MCMC_MH_old( proposal = current + delta prior_proposal = log_prior(proposal, AllPars) - if np.isfinite( - prior_proposal - ): # proposal satisfies the prior constraints + if np.isfinite(prior_proposal): # proposal satisfies the prior constraints # likelihood_proposal = log_likelihood(ParticleNum, proposal, variables, resultsLocation) likelihood_proposal, prediction_proposal = runFEM( ParticleNum, @@ -179,14 +175,10 @@ def MCMC_MH_old( log_acceptance = posterior_proposal - posterior_current all_proposals.append(proposal) - all_PLP.append( - [prior_proposal, likelihood_proposal, posterior_proposal] - ) + all_PLP.append([prior_proposal, likelihood_proposal, posterior_proposal]) # if np.isfinite(log_acceptance) and (np.log(np.random.uniform()) < log_acceptance): - if np.isfinite(log_acceptance) and ( - np.log(rng.uniform()) < log_acceptance - ): + if np.isfinite(log_acceptance) and (np.log(rng.uniform()) < log_acceptance): # accept current = proposal posterior_current = posterior_proposal @@ -242,9 +234,7 @@ def MCMC_MH( proposal = current + delta prior_proposal = log_prior(proposal, AllPars) - if np.isfinite( - prior_proposal - ): # proposal satisfies the prior constraints + if np.isfinite(prior_proposal): # proposal satisfies the prior constraints # likelihood_proposal = log_likelihood(ParticleNum, proposal, variables, resultsLocation) likelihood_proposal, prediction_proposal = runFEM( ParticleNum, @@ -272,14 +262,10 @@ def MCMC_MH( log_acceptance = posterior_proposal - posterior_current all_proposals.append(proposal) - all_PLP.append( - [prior_proposal, likelihood_proposal, posterior_proposal] - ) + all_PLP.append([prior_proposal, likelihood_proposal, posterior_proposal]) # if np.isfinite(log_acceptance) and (np.log(np.random.uniform()) < log_acceptance): - if np.isfinite(log_acceptance) and ( - np.log(rng.uniform()) < log_acceptance - ): + if np.isfinite(log_acceptance) and (np.log(rng.uniform()) < log_acceptance): # accept current = proposal posterior_current = posterior_proposal @@ -382,18 +368,14 @@ def compute_beta_evidence(beta, log_likelihoods, logFile, threshold=1.0): if dBeta < 1e-3: dBeta = 1e-3 - weights, cov_weights, std_weights = get_weights( - dBeta, log_likelihoods - ) + weights, cov_weights, std_weights = get_weights(dBeta, log_likelihoods) break weights, cov_weights, std_weights = get_weights(dBeta, log_likelihoods) beta = beta + dBeta if beta > 0.95: beta = 1 - log_evidence = logsumexp(dBeta * log_likelihoods) - np.log( - len(log_likelihoods) - ) + log_evidence = logsumexp(dBeta * log_likelihoods) - np.log(len(log_likelihoods)) try: ESS = int(1 / np.sum((weights / np.sum(weights)) ** 2)) diff --git a/modules/performUQ/common/uq_utilities.py b/modules/performUQ/common/uq_utilities.py index 622b1c37b..0443430f7 100644 --- a/modules/performUQ/common/uq_utilities.py +++ b/modules/performUQ/common/uq_utilities.py @@ -4,18 +4,20 @@ import subprocess import sys import traceback +from dataclasses import dataclass from multiprocessing.pool import Pool -from typing import Any, Optional, Union +from typing import Any, Union import numpy as np +import numpy.typing as npt import quoFEM_RV_models +import scipy.stats from ERAClasses.ERADist import ERADist from ERAClasses.ERANataf import ERANataf from numpy.typing import NDArray -import scipy.stats -import numpy.typing as npt -from dataclasses import dataclass +if quoFEM_RV_models not in sys.modules: + import quoFEM_RV_models def _copytree(src, dst, symlinks=False, ignore=None): @@ -34,9 +36,7 @@ def _copytree(src, dst, symlinks=False, ignore=None): ): shutil.copy2(s, d) except Exception as ex: - msg = ( - f"Could not copy {s}. The following error occurred: \n{ex}" - ) + msg = f"Could not copy {s}. The following error occurred: \n{ex}" return msg return "0" @@ -57,9 +57,7 @@ def _append_msg_in_out_file(msg, out_file_name: str = "ops.out"): msg += "\n" msg += "your model says...\n" msg += "........\n" + errmsg + "\n........ \n" - msg += "to read more, see " + os.path.join( - os.getcwd(), out_file_name - ) + msg += "to read more, see " + os.path.join(os.getcwd(), out_file_name) return msg @@ -81,9 +79,7 @@ def __init__( ignore_nans: bool = True, ) -> None: self.full_path_of_tmpSimCenter_dir = full_path_of_tmpSimCenter_dir - self.list_of_dir_names_to_copy_files_from = ( - list_of_dir_names_to_copy_files_from - ) + self.list_of_dir_names_to_copy_files_from = list_of_dir_names_to_copy_files_from self.list_of_rv_names = list_of_rv_names self.driver_filename = driver_filename self.length_of_results = length_of_results @@ -143,9 +139,7 @@ def _create_workdir(self, simulation_number: int) -> str: raise ModelEvaluationError(msg) return workdir - def _create_params_file( - self, sample_values: NDArray, workdir: str - ) -> None: + def _create_params_file(self, sample_values: NDArray, workdir: str) -> None: list_of_strings_to_write = [] list_of_strings_to_write.append(f"{self.num_rv}") for i, rv in enumerate(self.list_of_rv_names): @@ -176,7 +170,7 @@ def _execute_driver_file(self, workdir: str) -> None: ) returnStringList.append(f"The return code was {ex.returncode}") returnStringList.append(f"The following error occurred: \n{ex}") - raise ModelEvaluationError(f"\n\n".join(returnStringList)) + raise ModelEvaluationError("\n\n".join(returnStringList)) def _read_outputs_from_results_file(self, workdir: str) -> NDArray: if glob.glob("results.out"): @@ -303,9 +297,7 @@ def __init__( self.correlation_matrix_data, self.num_rvs ) self.marginal_ERAdistribution_objects_list = ( - make_list_of_marginal_distributions( - self.list_of_random_variables_data - ) + make_list_of_marginal_distributions(self.list_of_random_variables_data) ) self.ERANataf_object = make_ERANataf_object( self.marginal_ERAdistribution_objects_list, self.correlation_matrix @@ -316,7 +308,9 @@ def u_to_x( ) -> Union[tuple[NDArray[np.float64], Any], NDArray[np.float64]]: return self.ERANataf_object.U2X(U=u, Jacobian=jacobian) - def x_to_u(self, x: NDArray, jacobian: bool = False) -> Union[ + def x_to_u( + self, x: NDArray, jacobian: bool = False + ) -> Union[ tuple[NDArray[np.floating[Any]], NDArray[np.floating[Any]]], NDArray[np.floating[Any]], ]: @@ -347,9 +341,7 @@ def random( def get_list_of_pseudo_random_number_generators(entropy, num_spawn): seed_sequence = np.random.SeedSequence(entropy=entropy).spawn(num_spawn) - prngs = [ - np.random.Generator(np.random.PCG64DXSM(s)) for s in seed_sequence - ] + prngs = [np.random.Generator(np.random.PCG64DXSM(s)) for s in seed_sequence] return prngs @@ -399,9 +391,7 @@ def get_default_model_evaluation_function(model): return model.evaluate_model_once -def get_ERANataf_joint_distribution_instance( - list_of_rv_data, correlation_matrix_data -): +def get_ERANataf_joint_distribution_instance(list_of_rv_data, correlation_matrix_data): joint_distribution = ERANatafJointDistribution( list_of_rv_data, correlation_matrix_data ) @@ -454,9 +444,7 @@ def get_standard_normal_random_variates(list_of_prngs, size=1): def get_inverse_gamma_random_variate(prng, shape, scale, size=1): - return scipy.stats.invgamma.rvs( - shape, scale=scale, size=size, random_state=prng - ) + return scipy.stats.invgamma.rvs(shape, scale=scale, size=size, random_state=prng) def multivariate_normal_logpdf(x, mean, cov): From 43e1596251b873cafe51fe01c528c35c7a1d5194 Mon Sep 17 00:00:00 2001 From: bsaakash <11618528+bsaakash@users.noreply.github.com> Date: Tue, 13 Aug 2024 17:30:23 -0700 Subject: [PATCH 07/26] abs - adding back error suppression comment --- modules/performUQ/UCSD_UQ/UCSD_UQ.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/performUQ/UCSD_UQ/UCSD_UQ.py b/modules/performUQ/UCSD_UQ/UCSD_UQ.py index 937d06576..b6bf2bbfe 100644 --- a/modules/performUQ/UCSD_UQ/UCSD_UQ.py +++ b/modules/performUQ/UCSD_UQ/UCSD_UQ.py @@ -54,7 +54,7 @@ def main(args): # noqa: D103 err_file.touch() try: - result = subprocess.run( + result = subprocess.run( # noqa: S603 command_list, capture_output=True, text=True, From ffceb1a7cda413d2a70bc7ec1a02e8f409f56c0e Mon Sep 17 00:00:00 2001 From: bsaakash <11618528+bsaakash@users.noreply.github.com> Date: Tue, 13 Aug 2024 17:32:16 -0700 Subject: [PATCH 08/26] abs - removing unused code --- modules/performUQ/UCSD_UQ/parseData.py | 61 -------------------------- 1 file changed, 61 deletions(-) diff --git a/modules/performUQ/UCSD_UQ/parseData.py b/modules/performUQ/UCSD_UQ/parseData.py index b96c39173..23902ddf4 100644 --- a/modules/performUQ/UCSD_UQ/parseData.py +++ b/modules/performUQ/UCSD_UQ/parseData.py @@ -68,67 +68,6 @@ def parseDataFunction(dakotaJsonFile, logFile, tmpSimCenterDir, mainscriptDir): if 'parallelExecution' in uqInputs: parallelizeMCMC = uqInputs['parallelExecution'] # noqa: N806, F841 - # logFile.write("\n\t\t\tProcessing the log-likelihood script options") - # # If log-likelihood script is provided, use that, otherwise, use default log-likelihood function - # if ( - # len(logLikelihoodFile) > 0 - # ): # if the log-likelihood file is not an empty string - # logFile.write( - # "\n\t\t\t\tSearching for a user-defined log-likelihood script '{}'".format( - # logLikelihoodFile - # ) - # ) - # if os.path.exists(os.path.join(tmpSimCenterDir, logLikelihoodFile)): - # logFile.write( - # "\n\t\t\t\tFound log-likelihood file '{}' in {}.".format( - # logLikelihoodFile, tmpSimCenterDir - # ) - # ) - # logLikeModuleName = os.path.splitext(logLikelihoodFile)[0] - # try: - # import_module(logLikeModuleName) - # except: - # logFile.write( - # "\n\t\t\t\tERROR: The log-likelihood script '{}' cannot be imported.".format( - # os.path.join(tmpSimCenterDir, logLikelihoodFile) - # ) - # ) - # raise - # else: - # logFile.write( - # "\n\t\t\t\tERROR: The log-likelihood script '{}' cannot be found in {}.".format( - # logLikelihoodFile, tmpSimCenterDir - # ) - # ) - # raise FileNotFoundError( - # "ERROR: The log-likelihood script '{}' cannot be found in {}.".format( - # logLikelihoodFile, tmpSimCenterDir - # ) - # ) - # else: - # defaultLogLikeFileName = "defaultLogLikeScript.py" - # defaultLogLikeDirectoryPath = mainscriptDir - # sys.path.append(defaultLogLikeDirectoryPath) - # logLikeModuleName = os.path.splitext(defaultLogLikeFileName)[0] - # logFile.write("\n\t\t\t\tLog-likelihood script not provided.") - # logFile.write( - # "\n\t\t\t\tUsing the default log-likelihood script: \n\t\t\t\t\t{}".format( - # os.path.join( - # defaultLogLikeDirectoryPath, defaultLogLikeFileName - # ) - # ) - # ) - # try: - # import_module(logLikeModuleName) - # except: - # logFile.write( - # "\n\t\t\t\tERROR: The log-likelihood script '{}' cannot be imported.".format( - # os.path.join(tmpSimCenterDir, logLikelihoodFile) - # ) - # ) - # raise - # logLikeModule = import_module(logLikeModuleName) - # Processing EDP inputs logFile.write('\n\n\t\tProcessing EDP inputs') edpNamesList = [] # noqa: N806 From fd7c421d691f7cece448f71b40c3982c8558aa4b Mon Sep 17 00:00:00 2001 From: Sina Naeimi Date: Wed, 14 Aug 2024 09:07:56 -0700 Subject: [PATCH 09/26] SN - PyReCoDes-REWET interface wrapper added --- .../pyrecodes/PyReCode-REWET-preprocessor.py | 565 ++++++++++++++++++ .../performREC/pyrecodes/damage_convertor.py | 395 ++++++++++++ modules/performREC/pyrecodes/rewet_helper.py | 9 + 3 files changed, 969 insertions(+) create mode 100644 modules/performREC/pyrecodes/PyReCode-REWET-preprocessor.py create mode 100644 modules/performREC/pyrecodes/damage_convertor.py create mode 100644 modules/performREC/pyrecodes/rewet_helper.py diff --git a/modules/performREC/pyrecodes/PyReCode-REWET-preprocessor.py b/modules/performREC/pyrecodes/PyReCode-REWET-preprocessor.py new file mode 100644 index 000000000..135d20e85 --- /dev/null +++ b/modules/performREC/pyrecodes/PyReCode-REWET-preprocessor.py @@ -0,0 +1,565 @@ +# -*- coding: utf-8 -*- + +import random +import json +import copy +import rewet +from pathlib import Path +import wntrfr +import pandas as pd +from sklearn.cluster import KMeans +from rewet.api import API +import rewet_helper as rh + +TEMP_DIR = "./" +RESULT_DIR = "./rewet_result" +INPUT_FILE_DIR = "./" + + +class REWETPyReCoDes(): + def __init__(self): + self.wn = None + self._clean_wn = None + self.inp_file_path = None + self.asset_information = {} + self.building_coordinates = [] + self.demand_node_to_building = {} + self.buildings = {} + self.nodes = {} + self.pipe_damage = None + self.node_damage = None + self.pump_damage = None + self.tank_damage = None + self.rewet = None + self.damage_file_list = {} + self.damage_state = {} + self.hydraulic_time_step = 3600 + + def system_state(self, state, damage, inp_file, damage_time=0): + """ + Sets the WDN system for PyReCoDes Interface. + + Parameters + ---------- + state : dict + The State of system which is defiend in the R2DTool's system_det + style. + damage : dict + The damage state in 2DTool's system_i style. + inp_file : path(str) + the path to the inp file. + Damage_time : int + When initil damages happen in seconds. + + Returns + ------- + None. + + """ + + # read the inp file + self.read_inp_file(inp_file) + + self.set_asset_data(state) + + # sets the damage state based on the initial damage (from pelicun) + self.set_initial_water_damage_state(damage, damage_time) + + # couple building to the demand nodes + self.couple_buildings_to_demand_nodes(state) + + + + def system_performance(self, + state, + damage, + current_time, + next_time): + + self.wn = copy.deepcopy(self._clean_wn) + # sets the damage state based on the current (change of the network) + # and saves the current time + self.save_damage(damage, current_time) + + # prepare rewet inputs + self.make_rewet_inputs(current_time, next_time) + + # load REWET API interface + self.rewet = API(self.input_file_path) + + # sets the new demand absed on the new percentage + self.set_new_demand(state) + + # apply_damages + ## self.apply_damage() + + # run WDN performance evaluation + self.run_performance(current_time, next_time) + + # Save REWET Result + self.rewet.save_result() + + # Get result + building_satisfaction = self.get_building_data_satisfaction( + method="mean") + self.building_satisfaction = building_satisfaction + + return building_satisfaction + + def read_inp_file(self, inp_file): + """ + Reads the inp file + + Parameters + ---------- + inp_file : str + The path to the inp file. + + Returns + ------- + None. + + """ + + self.inp_file_path = Path(inp_file) + self.inp_file_path = self.inp_file_path.resolve() + + if not self.inp_file_path.exists(): + raise ValueError(f"There inp file does not exists: {inp_file}") + + self.inp_file_path = str(self.inp_file_path) + + # read the inp file and create the WDN obejct file + self.wn = wntrfr.network.model.WaterNetworkModel(self.inp_file_path) + self._clean_wn = copy.deepcopy(self.wn) + + for node_name, node in self.wn.junctions(): + node_demand_base_value = node.demand_timeseries_list[0].base_value + if node_demand_base_value > 0: + if node_name not in self.nodes: + self.nodes[node_name] = {} + + self.nodes[node_name]["initial_demand"] = \ + node_demand_base_value + + self.nodes[node_name]["coordinates"] = node.coordinates + + + def set_asset_data(self, state): + wdn_state = state["WaterDistributionNetwork"] + wdn_state = wdn_state.get("Pipe", []) + + for asset_type, asset_type_data in state.items(): + if asset_type not in self.asset_information: + # check if asset_type exists in self.asset_information + self.asset_information[asset_type] = {} + for sub_asset_type, sub_asset_type_data in asset_type_data.items(): + # check if sub_asset_type exists in + # self.asset_information[asset_type] + if sub_asset_type not in self.asset_information[asset_type]: + self.asset_information[asset_type][sub_asset_type] = {} + + for element_key, element_data in sub_asset_type_data.items(): + asset_id = element_data["GeneralInformation"]["AIM_id"] + if asset_id != element_key: + raise ValueError("The rationality behidn the workdflow" + "is that oth aim-id and keys be the" + "same") + + self.asset_information[asset_type]\ + [sub_asset_type]\ + [asset_id] = element_data["GeneralInformation"] + + building_state = state["Buildings"]["Building"] + + for building_id, each_building in building_state.items(): + population = each_building["GeneralInformation"]["Population"] + population_ratio = each_building.get("Population_Ratio", None) + + if population_ratio is not None: + ratio = population_ratio + else: + ratio = 1 + + cur_building = {} + cur_building["initial_population_ratio"] = ratio + cur_building["population_ratio"] = ratio + cur_building["initial_population"] = population + cur_building["population"] = population + + self.buildings[building_id] = cur_building + + + def set_rewet_damage(self, damage, damage_time): + damage = damage["WaterDistributionNetwork"] + pipe_damage = damage.get("Pipe", []) + + damage_list = [] + for asset_id, damage_location in pipe_damage.items(): + damage_location_info = damage_location["Damage"] + + aggregate_keys =[ + key for key in damage_location_info + if "aggregate-" in key] + + aggregate_keys.sort() + + aggregate_results = [damage_location_info[key] + for key in aggregate_keys] + + segment_sizes = len(aggregate_results ) + segment_step = 1 / segment_sizes + c = 0 + + for damage_val in aggregate_results: + if damage_val > 0: + if damage_val == 1: + damage_type = "leak" + elif damage_val == 2: + damage_type = "break" + else: + raise ValueError("The damage type must be eother " + "1 or 2") + else: + continue + + + cur_loc = c * segment_step + segment_step / 2 + + c += 1 + + pipe_id = self.asset_information["WaterDistributionNetwork"]\ + ["Pipe"][asset_id]["InpID"] + damage_list.append( {"pipe_id": pipe_id, "damage_loc": cur_loc, + "type": damage_type, "Material": "CI"} + ) + damage_list.reverse() + self.pipe_damage = pd.Series(data=damage_list, + index=[damage_time for val in + damage_list], dtype="O") + + self.node_damage = pd.Series(dtype="O") + + self.pump_damage = pd.Series(dtype="O") + + self.tank_damage = pd.Series(dtype="O") + + + def set_initial_water_damage_state(self, damage, damage_time): + self.damage_state[damage_time] = damage + self.set_rewet_damage(self.damage_state[damage_time], damage_time) + + def couple_buildings_to_demand_nodes(self, state): + building_state = state["Buildings"]["Building"] + + building_id_list = [] + for building_id, each_building in building_state.items(): + location = each_building["GeneralInformation"]["location"] + coordinate = (location["latitude"], location["longitude"]) + building_id_list.append(building_id) + + self.building_coordinates.append(coordinate) + + + demand_node_coordinate_list = [val["coordinates"] for key, val in + self.nodes.items()] + + demand_node_name_list = [key for key, val in + self.nodes.items()] + + kmeans = KMeans(n_clusters=len(demand_node_coordinate_list), + init=demand_node_coordinate_list, + n_init=1, + random_state=0) + + kmeans.fit(self.building_coordinates) + + labels = kmeans.labels_ + labels = labels.tolist() + + for group_i in range(len(demand_node_coordinate_list)): + node_name = demand_node_name_list[group_i] + for building_l in range(len(labels)): + cur_node_l = labels[building_l] + if group_i == cur_node_l: + if node_name not in self.demand_node_to_building: + self.demand_node_to_building[node_name] = [] + + building_id = building_id_list[building_l] + self.demand_node_to_building[node_name].append(building_id) + + + for node_name in self.demand_node_to_building: + building_name_list = self.demand_node_to_building[node_name] + population_list = [self.buildings[bldg_id]["initial_population"] + for bldg_id in building_name_list] + + total_initial_population = sum(population_list) + + cur_node = self.nodes[node_name] + initial_node_demand = cur_node["initial_demand"] + + if initial_node_demand == 0 and total_initial_population > 0: + Warning(f"Initial demand for node {node_name} is 0." + + "Thus, the demand ratio in buildidng(s)" + + "{repr(building_name_list).strip('[').strip(']'))}" + + "is naturally ineffective and their demand is not met." + ) + + if total_initial_population == 0: + Warning(f"The population assigned to {node_name} is 0." + + "Thus, the demand ratio in buildidng(s)" + + "{repr(building_name_list).strip('[').strip(']'))}" + + "is ignored." + ) + # We assume that population in State does not change in teh course + # of recovery. Thus, the population is intiial population. For more + # clarity, we name the variable "initial_population". + # It does not mean that there will be ffdifferent population in + # the course of recovery + self.nodes[node_name]["initial_population"] = \ + total_initial_population + + self.nodes[node_name]["initial_node_demand"] = initial_node_demand + + for bldg_id in building_name_list: + pop = self.buildings[bldg_id]["initial_population"] + + if total_initial_population != 0: + cur_bldg_initial_demand = \ + pop / total_initial_population * initial_node_demand + else: + cur_bldg_initial_demand = None + + self.buildings[bldg_id]["initial_demand"] = \ + cur_bldg_initial_demand + + def save_damage(self, damage, current_time): + """ + Convert and save the dmaages that are set before + + Parameters + ---------- + damage : dict + damage dict. + current_time : int + Current time. + + Returns + ------- + None. + + """ + pipe_damage_file_name = "temp_pipe_damage_file.pkl" + node_damage_file_name = "node_pipe_damage_file.pkl" + tank_damage_file_name = "tank_pipe_damage_file.pkl" + pump_damage_file_name = "pump_pipe_damage_file.pkl" + + pipe_path = Path(TEMP_DIR) / pipe_damage_file_name + node_path = Path(TEMP_DIR) / node_damage_file_name + tank_path = Path(TEMP_DIR) / tank_damage_file_name + pump_path = Path(TEMP_DIR) / pump_damage_file_name + list_path = Path(TEMP_DIR) / "list.xlsx" + + pipe_path = str(pipe_path) + node_path = str(node_path) + tank_path = str(tank_path) + pump_path = str(pump_path) + + self.damage_state[current_time] = damage + self.set_rewet_damage(self.damage_state[current_time], current_time) + + self.pipe_damage.to_pickle(pipe_path) + self.node_damage.to_pickle(node_path) + self.pump_damage.to_pickle(pump_path) + self.tank_damage.to_pickle(tank_path) + + scn_postfix_list = random.choices( + ["A","B","C","D","E","F","G","H","I","J","L","M"], + k=0) + + scn_postfix = "" + for p in scn_postfix_list: + scn_postfix += p + + self.damage_file_list["Scenario Name"] = "SCN_" + scn_postfix + self.damage_file_list["Pipe Damage"] = pipe_damage_file_name + self.damage_file_list["Nodal Damage"] = node_damage_file_name + self.damage_file_list["Pump Damage"] = tank_damage_file_name + self.damage_file_list["Tank Damage"] = pump_damage_file_name + self.damage_file_list["Probability"] = 1 + + damage_file_list = pd.DataFrame.from_dict([self.damage_file_list]) + damage_file_list.to_excel(list_path) + self.list_path = list_path + + def run_performance(self, current_time, next_time): + + """ + Runs the performance model using a hydraic solver (REWET) + + Parameters + ---------- + current_time : int + Current time in day. + next_time : int + Mext time in day. + + Raises + ------ + ValueError + When the current or next time is given wrong. + + Returns + ------- + building_demand : dict + the result specifying the percentage of demand satisfied by each + building. + + """ + + if current_time > next_time: + raise ValueError("Current tiime cannot be bigger than the next" + "time") + if abs(next_time - current_time) % self.hydraulic_time_step != 0: + raise ValueError("next_time - current_time must be a factor of " + "the hydraulci time step.") + + status = self.rewet.initiate(current_time=current_time, debug=True) + if status != 0: + raise ValueError(f"There is an error: {status}") + + self.rewet.apply_damage(current_time, 0.001) + + time_step = next_time - current_time + status = self.rewet.run_hydraulic_simulation(time_step) + if status != 0: + raise ValueError(f"There is an error: {status}") + + + return dict + + def make_rewet_inputs(self, current_time, next_time): + settings = rh.get_rewet_hydraulic_basic_setting() + run_time = next_time - current_time + list_file_path = Path(self.list_path) + list_file_path = list_file_path.resolve() + if not list_file_path.exists(): + raise ValueError(f"The list file does not exists: " + f"{str(list_file_path)}") + list_file_path = str(list_file_path) + + temp_dir = Path(TEMP_DIR).resolve() + if not temp_dir.exists(): + raise ValueError(f"The temp directory does not exists: " + f"{str(temp_dir)}") + temp_dir = str(temp_dir) + + settings["RUN_TIME"] = run_time + settings["minimum_simulation_time"] = run_time + settings["result_directory"] = RESULT_DIR + settings["temp_directory"] = temp_dir + settings["WN_INP"] = self.inp_file_path + settings['pipe_damage_file_list'] = list_file_path + settings['pipe_damage_file_directory'] = temp_dir + settings['Restoration_on'] = False + settings['Pipe_damage_input_method'] = "pickle" + + input_file_path = Path(INPUT_FILE_DIR) / "rewet_input.json" + input_file_path = input_file_path.resolve() + with open(input_file_path, "wt") as f: + json.dump(settings, f, indent=4) + + self.input_file_path = str(input_file_path) + + def get_building_data_satisfaction(self, method): + + demand_sat = self.rewet.get_satisfied_demand_ratio() + + if method.upper() == "MEAN": + demand_sat = demand_sat.mean() + elif method.upper() == "MAX": + demand_sat = demand_sat.max() + elif method.upper() == "MIN": + demand_sat = demand_sat.min() + else: + raise ValueError(f"The method is not recognizable: {method}") + + building_demand_satisfaction_ratio = {} + + for node_name in self.demand_node_to_building: + node_demand_ratio = demand_sat[node_name] + building_names = self.demand_node_to_building[node_name] + cur_satisified_demand_building = dict(zip( + building_names, + [node_demand_ratio]*len(building_names))) + + building_demand_satisfaction_ratio.update( + cur_satisified_demand_building) + + return building_demand_satisfaction_ratio + + + + def set_new_demand(self, state): + + for node_name in self.demand_node_to_building: + + cur_node = self.nodes[node_name] + total_initial_population = \ + self.nodes[node_name]["initial_population"] + + if not total_initial_population > 0: + continue + + building_name_list = self.demand_node_to_building[node_name] + + building = state["Buildings"]["Building"] + + node_new_demand = 0 + + for bldg_id in building_name_list: + + cur_bldg_initial_demand = \ + self.buildings[bldg_id]["initial_demand"] + + cur_bldg_deamnd_ratio = \ + building[bldg_id]["GeneralInformation"]["Population_Ratio"] + + cur_bldg_new_deamnd = \ + cur_bldg_deamnd_ratio * cur_bldg_initial_demand + + self.buildings[bldg_id]["current_demand"] = \ + cur_bldg_new_deamnd + + node_new_demand += cur_bldg_new_deamnd + + self.nodes[node_name]["current_demand"] = node_new_demand + node = self.wn.get_node(node_name) + node.demand_timeseries_list[0].base_value = node_new_demand + + + +if __name__ == '__main__': + # raise RuntimeError("This file should not be run") + with open("Results_det.json", "rt") as f: + state = json.load(f) + + with open("Results_0.json", "rt") as f: + damage = json.load(f) + + inp_file = "waterNetwork.inp" + + interface = REWETPyReCoDes() + interface.system_state(state, damage, inp_file) + result = interface.system_performance(state, + damage, + 0, + 24*3600) + + # = interface.system_performance(state, + #damage, + #1*24*3600, + #2*24*3600) + + diff --git a/modules/performREC/pyrecodes/damage_convertor.py b/modules/performREC/pyrecodes/damage_convertor.py new file mode 100644 index 000000000..d460bbbcb --- /dev/null +++ b/modules/performREC/pyrecodes/damage_convertor.py @@ -0,0 +1,395 @@ +# -*- coding: utf-8 -*- +""" +Created on Mon Jan 8 09:40:15 2024 + +@author: naeim +""" + +import os +from pathlib import Path +import pandas as pd +import preprocessorIO + +CBIG_int = int(1e9) + + + + +def createPipeDamageInputForREWET(pipe_damage_data, run_dir, event_time, sc_geojson): + """ + Creates REWET-style piep damage file. + + Parameters + ---------- + pipe_damage_data : dict + Pipe damage data from PELICUN. + REWET_input_data : dict + REWET input data. + + Raises + ------ + ValueError + If damage type is not what it should be. + + Returns + ------- + pipe_damage_list : Pandas Series + REWET-style pipe damage file. + + """ + pipe_id_list = [key for key in pipe_damage_data] + + damage_list = [] + damage_time = event_time + sc_geojson_file = preprocessorIO.readJSONFile(sc_geojson) + pipe_data = [ss for ss in sc_geojson_file["features"] if ss["properties"]["type"]=="Pipe"] + pipe_index = [str(ss["id"]) for ss in pipe_data] + pipe_id = [ss["properties"]["InpID"] for ss in pipe_data] + pipe_index_to_id = dict(zip(pipe_index, pipe_id)) + + for pipe_id in pipe_id_list: + cur_data = pipe_damage_data[pipe_id] + + cur_damage = cur_data["Damage"] + cur_demand = cur_data["Demand"] + + aim_data = findAndReadAIMFile(pipe_id,os.path.join( + "Results", "WaterDistributionNetwork", "Pipe"), run_dir) + + material = aim_data["GeneralInformation"].get("Material", None) + + if material == None: + #raise ValueError("Material is none") + material = "CI" + + aggregates_list = [cur_agg for cur_agg in list( cur_damage.keys() ) if "aggregate" in cur_agg] + segment_sizes = len(aggregates_list ) + segment_step = 1 / segment_sizes + c = 0 + + for cur_agg in aggregates_list: #cur_damage["aggregate"]: + damage_val = cur_damage[cur_agg] + if damage_val > 0: + if damage_val == 1: + damage_type = "leak" + elif damage_val == 2: + damage_type = "break" + else: + raise ValueError("The damage type must be eother 1 or 2") + else: + continue + + + cur_loc = c * segment_step + segment_step / 2 + #print(cur_loc) + c += 1 + damage_list.append( {"pipe_id": pipe_index_to_id[pipe_id], "damage_loc": cur_loc, + "type": damage_type, "Material": material} + ) + damage_list.reverse() + pipe_damage_list = pd.Series(data=damage_list, + index=[damage_time for val in damage_list], dtype="O") + + #REWET_input_data["Pipe_damage_list"] = pipe_damage_list + #REWET_input_data["AIM"] = aim_data + + + return pipe_damage_list + +def createNodeDamageInputForREWET(node_damage_data, run_dir, event_time): + """ + Creates REWET-style node damage file. + + Parameters + ---------- + node_damage_data : dict + Node damage data from PELICUN. + REWET_input_data : dict + REWET input data. + + Returns + ------- + node_damage_list : Pandas Series + REWET-style node damage file. + + """ + node_id_list = [key for key in node_damage_data] + + damage_list = [] + damage_time = event_time + + for node_id in node_id_list: + cur_damage = node_damage_data[node_id] + aggregates_list = [cur_agg for cur_agg in list( cur_damage.keys() ) if "aggregate" in cur_agg] + + if len(aggregates_list) == 0: + continue + + cur_data = node_damage_data[node_id] + + cur_damage = cur_data["Damage"] + cur_demand = cur_data["Demand"] + + aim_data = findAndReadAIMFile(node_id,os.path.join( + "Results", "WaterDistributionNetwork", "Node"), + run_dir) + + total_length = aim_data["GeneralInformation"].get("Total_length", None) + total_number_of_damages = cur_damage["aggregate"] + + damage_list.append( {"node_name": node_id, + "number_of_damages": total_number_of_damages, + "node_Pipe_Length": total_length} + ) + + node_damage_list = pd.Series(data=damage_list, + index=[damage_time for val in damage_list], dtype="O") + + return node_damage_list + +def createPumpDamageInputForREWET(pump_damage_data, REWET_input_data): + """ + Creates REWET-style pump damage file. + + Parameters + ---------- + pump_damage_data : dict + Pump damage data from PELICUN. + REWET_input_data : dict + REWET input data. + + Returns + ------- + pump_damage_list : Pandas Series + REWET-style pump damage file. + + """ + pump_id_list = [key for key in pump_damage_data] + + damage_list = [] + damage_time = REWET_input_data["event_time"] + + for pump_id in pump_id_list: + cur_data = pump_damage_data[pump_id] + + cur_damage = cur_data["Damage"] + cur_repair_time = cur_data["Repair"] + + if cur_damage == 0: + continue # cur_damage_state = 0 means undamaged pump + + # I'm not sure if we need any data about the pump at this point + + #aim_data = findAndReadAIMFile(tank_id, os.path.join( + #"Results", "WaterDistributionNetwork", "Pump"), + #REWET_input_data["run_dir"]) + + #We are getting this data from PELICUN + #restore_time = getPumpRetsoreTime(cur_damage) + damage_list.append( {"pump_id": pump_id, + "time": damage_time, "Restore_time": cur_repair_time} + ) + pump_damage_list = pd.Series(index=[damage_time for val in damage_list], data=damage_list) + + return pump_damage_list + + + +def createTankDamageInputForREWET(tank_damage_data, REWET_input_data): + """ + Creates REWET-style Tank damage file. + + Parameters + ---------- + tank_damage_data : dict + Tank damage data from PELICUN. + REWET_input_data : dict + REWET input data. + + Returns + ------- + tank_damage_list : Pandas Series + REWET-style tank damage file. + """ + tank_id_list = [key for key in tank_damage_data] + + damage_list = [] + damage_time = REWET_input_data["event_time"] + + for tank_id in tank_id_list: + cur_data = tank_damage_data[tank_id] + + cur_damage = cur_data["Damage"] + cur_repair_time = cur_data["Repair"] + + if cur_damage == 0: + continue # cur_damage_state = 0 meeans undamged tank + +# ============================================================================= +# # We are getting his data from REWET +# +# aim_data = findAndReadAIMFile(tank_id, os.path.join( +# "Results", "WaterDistributionNetwork", "Tank"), +# REWET_input_data["run_dir"]) +# tank_type = aim_data["GeneralInformation"].get("Type", None) +# restore_time = getTankRetsoreTime(tank_type, cur_damage) +# ============================================================================= + + damage_list.append( {"tank_id": tank_id, + "time": damage_time, "Restore_time": cur_repair_time} + ) + + tank_damage_list = pd.Series(index=[damage_time for val in damage_list], data=damage_list) + + return tank_damage_list + + +def findAndReadAIMFile(asset_id, asset_type, run_dir): + """ + Finds and read the AIM file for an asset. + + Parameters + ---------- + asset_id : int + The asset ID. + asset_type : str + Asset Type (e.g., Building, WaterDistributionNetwork). + run_dir : path + The directory where data is stored (aka the R2dTool directory) + + Returns + ------- + aim_file_data : dict + AIM file data as a dict. + + """ + + file_path = Path(run_dir, asset_type, str(asset_id), "templatedir", f"{asset_id}-AIM.json") + aim_file_data = preprocessorIO.readJSONFile(str(file_path) ) + return aim_file_data + +def getPumpRetsoreTime(damage_state): + """ + NOT USED! WE WILL GET IT FROM PELICUN + + Provides the restore time based on HAZUS repair time or any other + approach available in the future. If damage state is slight, the restore + time is 3 days (in seconds). If damage state is 2, the restore time is 7 + days (in seconds). If damage state is 3 or 4, the restore time is + indefinite (a big number). + + Parameters + ---------- + damage_state : Int + Specifies the damage state (1 for slightly damages, 2 for moderate, + 3 etensive, and 4 complete. + + Returns + ------- + Retstor time : int + + + """ + + if damage_state == 1: + restore_time = int(3 * 24 * 3600) + elif damage_state == 2: + restore_time = int(7 * 24 * 3600) + else: + restore_time = CBIG_int + + return restore_time + +def getTankRetsoreTime(tank_type, damage_state): + """ + NOT USED! WE WILL GET IT FROM PELICUN + + Provides the restore time based on HAZUS repair time or any other + approach available in the future. if damage state is slight, the restore + time is 3 days (in seconds). If damage state is 2, the restore time is 7 + days (in seconds). If damage state is 3 or 4, the restore time is + indefinite (a big number). + + Parameters + ---------- + tank_type : STR + Tank type based on the data schema. The parametr is not used for now. + damage_state : Int + Specifies the damage state (1 for slightly damages, 2 for moderate, + 3 etensive, and 4 complete. + + Returns + ------- + Retstor time : int + + + """ + + if damage_state == 1: + restore_time = int(3 * 24 * 3600) + elif damage_state == 2: + restore_time = int(7 * 24 * 3600) + else: + restore_time = CBIG_int + + return restore_time + +def readDamagefile(file_addr, run_dir, event_time, sc_geojson): + """ + Reads PELICUN damage files and create REWET-Style damage for all + WaterDistributionNetwork elements + + Parameters + ---------- + file_addr : path + PELICUN damage file in JSON format. + REWET_input_data : dict + REWET input data, whcih is updated in the function. + scn_number : dict + JSON FILE. + + Returns + ------- + damage_data : dict + Damage data in PELICUN dict format. + + """ + # TODO: Make reading once for each scneario + + #wn = wntrfr.network.WaterNetworkModel(REWET_input_data["inp_file"] ) + + damage_data = preprocessorIO.readJSONFile(file_addr) + + wn_damage_data = damage_data["WaterDistributionNetwork"] + + if "Pipe" in wn_damage_data: + pipe_damage_data = createPipeDamageInputForREWET( + wn_damage_data["Pipe"], run_dir, event_time, sc_geojson) + else: + pipe_damage_data = pd.Series(dtype="O") + + if "Tank" in wn_damage_data: + tank_damage_data = createTankDamageInputForREWET( + wn_damage_data["Tank"], run_dir, event_time) + else: + tank_damage_data = pd.Series(dtype="O") + + if "Pump" in wn_damage_data: + pump_damage_data = createPumpDamageInputForREWET( + wn_damage_data["Pump"], run_dir, event_time) + else: + pump_damage_data = pd.Series(dtype="O") + + if "Junction" in wn_damage_data: + node_damage_data = createNodeDamageInputForREWET( + wn_damage_data["Junction"], run_dir, event_time) + else: + node_damage_data = pd.Series(dtype="O") + + damage_data = {} + damage_data["Pipe"] = pipe_damage_data + damage_data["Tank"] = tank_damage_data + damage_data["Pump"] = pump_damage_data + damage_data["Node"] = node_damage_data + + return damage_data \ No newline at end of file diff --git a/modules/performREC/pyrecodes/rewet_helper.py b/modules/performREC/pyrecodes/rewet_helper.py new file mode 100644 index 000000000..9f3718635 --- /dev/null +++ b/modules/performREC/pyrecodes/rewet_helper.py @@ -0,0 +1,9 @@ +import rewet.Input.Settings + + +def get_rewet_hydraulic_basic_setting(): + settings = rewet.Input.Settings.Settings() + settings_dict = settings.process.settings + # settings_dict.update(settings.scenario.settings) + + return settings_dict From 15d24ff178f37fddd260927a59722ee560df2c57 Mon Sep 17 00:00:00 2001 From: bsaakash <11618528+bsaakash@users.noreply.github.com> Date: Thu, 15 Aug 2024 15:00:32 -0700 Subject: [PATCH 10/26] abs - fixing issues highlighted by linter --- modules/performUQ/UCSD_UQ/parseData.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/performUQ/UCSD_UQ/parseData.py b/modules/performUQ/UCSD_UQ/parseData.py index 1dbdc7f5b..d37938197 100644 --- a/modules/performUQ/UCSD_UQ/parseData.py +++ b/modules/performUQ/UCSD_UQ/parseData.py @@ -23,7 +23,7 @@ def __init__(self, message): self.message = message -def parseDataFunction(dakotaJsonFile, logFile, tmpSimCenterDir, mainscriptDir): # noqa: C901, N802, N803, D103, PLR0915 +def parseDataFunction(dakotaJsonFile, logFile): # noqa: C901, N802, N803, D103, PLR0915 # Read in the json object logFile.write('\n\tReading the json file') with open(dakotaJsonFile) as f: # noqa: PTH123 @@ -61,7 +61,7 @@ def parseDataFunction(dakotaJsonFile, logFile, tmpSimCenterDir, mainscriptDir): maxRunTime = uqInputs['maxRunTime'] # noqa: N806 else: maxRunTime = float('inf') # noqa: N806, F841 - logLikelihoodFile = uqInputs['logLikelihoodFile'] # noqa: N806 + # logLikelihoodFile = uqInputs['logLikelihoodFile'] calDataFile = uqInputs['calDataFile'] # noqa: N806 parallelizeMCMC = True # noqa: N806 From 677cbbc08b82289ec9cbd51d568907f4c3cd5918 Mon Sep 17 00:00:00 2001 From: bsaakash <11618528+bsaakash@users.noreply.github.com> Date: Mon, 26 Aug 2024 12:59:19 -0700 Subject: [PATCH 11/26] abs - fixing import linter warning --- .../performUQ/UCSD_UQ/preprocess_hierarchical_bayesian.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/modules/performUQ/UCSD_UQ/preprocess_hierarchical_bayesian.py b/modules/performUQ/UCSD_UQ/preprocess_hierarchical_bayesian.py index 8b88e340a..a5eefc16e 100644 --- a/modules/performUQ/UCSD_UQ/preprocess_hierarchical_bayesian.py +++ b/modules/performUQ/UCSD_UQ/preprocess_hierarchical_bayesian.py @@ -7,9 +7,8 @@ import numpy as np -path_to_common_uq = Path(__file__).parent.parent / 'common' -sys.path.append(str(path_to_common_uq)) -import uq_utilities # noqa: E402 +sys.path.append(str(Path(__file__).parent.parent / 'common')) +import uq_utilities InputsType = tuple[ Path, From e1e6ef9cd95b5394523d421619d62f3006e463aa Mon Sep 17 00:00:00 2001 From: Sina Naeimi Date: Mon, 26 Aug 2024 14:36:00 -0700 Subject: [PATCH 12/26] Modified REWET-PyReCoDes interface. Tested --- ...rocessor.py => PyReCode-REWET-preprocessor - Copy.py} | 0 modules/performREC/pyrecodes/rewet_helper.py | 9 --------- 2 files changed, 9 deletions(-) rename modules/performREC/pyrecodes/{PyReCode-REWET-preprocessor.py => PyReCode-REWET-preprocessor - Copy.py} (100%) delete mode 100644 modules/performREC/pyrecodes/rewet_helper.py diff --git a/modules/performREC/pyrecodes/PyReCode-REWET-preprocessor.py b/modules/performREC/pyrecodes/PyReCode-REWET-preprocessor - Copy.py similarity index 100% rename from modules/performREC/pyrecodes/PyReCode-REWET-preprocessor.py rename to modules/performREC/pyrecodes/PyReCode-REWET-preprocessor - Copy.py diff --git a/modules/performREC/pyrecodes/rewet_helper.py b/modules/performREC/pyrecodes/rewet_helper.py deleted file mode 100644 index 9f3718635..000000000 --- a/modules/performREC/pyrecodes/rewet_helper.py +++ /dev/null @@ -1,9 +0,0 @@ -import rewet.Input.Settings - - -def get_rewet_hydraulic_basic_setting(): - settings = rewet.Input.Settings.Settings() - settings_dict = settings.process.settings - # settings_dict.update(settings.scenario.settings) - - return settings_dict From 263dc202ebdd9463baf7f2274bd2b0d44edae0ac Mon Sep 17 00:00:00 2001 From: Sina Naeimi Date: Mon, 26 Aug 2024 17:28:06 -0700 Subject: [PATCH 13/26] PEP8 compatibility checked --- ...Copy.py => PyReCode-REWET-preprocessor.py} | 385 ++++++++++++------ .../performREC/pyrecodes/damage_convertor.py | 341 ++++++++++------ 2 files changed, 476 insertions(+), 250 deletions(-) rename modules/performREC/pyrecodes/{PyReCode-REWET-preprocessor - Copy.py => PyReCode-REWET-preprocessor.py} (58%) diff --git a/modules/performREC/pyrecodes/PyReCode-REWET-preprocessor - Copy.py b/modules/performREC/pyrecodes/PyReCode-REWET-preprocessor.py similarity index 58% rename from modules/performREC/pyrecodes/PyReCode-REWET-preprocessor - Copy.py rename to modules/performREC/pyrecodes/PyReCode-REWET-preprocessor.py index 135d20e85..6a097efdb 100644 --- a/modules/performREC/pyrecodes/PyReCode-REWET-preprocessor - Copy.py +++ b/modules/performREC/pyrecodes/PyReCode-REWET-preprocessor.py @@ -1,4 +1,39 @@ -# -*- coding: utf-8 -*- +# Copyright (c) 2024 The Regents of the University of California +# Copyright (c) 2024 Leland Stanford Junior University +# +# This file is part of whale. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# You should have received a copy of the BSD 3-Clause License along with +# whale. If not, see . +# +# Contributors: +# Sina Naeimi import random import json @@ -9,14 +44,13 @@ import pandas as pd from sklearn.cluster import KMeans from rewet.api import API -import rewet_helper as rh TEMP_DIR = "./" RESULT_DIR = "./rewet_result" INPUT_FILE_DIR = "./" -class REWETPyReCoDes(): +class REWETPyReCoDes: def __init__(self): self.wn = None self._clean_wn = None @@ -63,23 +97,37 @@ def system_state(self, state, damage, inp_file, damage_time=0): self.set_asset_data(state) # sets the damage state based on the initial damage (from pelicun) - self.set_initial_water_damage_state(damage, damage_time) + self.update_state_with_damages(damage, damage_time, state) # couple building to the demand nodes self.couple_buildings_to_demand_nodes(state) + def system_performance(self, state, damage, current_time, next_time): + """ + Assesses the system functionality. + Parameters + ---------- + state : dict. + The _det file content. + damage : dict + _i (realization) file content. + current_time : int + Current time in seconds. + next_time : int + Next time in seconds. - def system_performance(self, - state, - damage, - current_time, - next_time): + Returns + ------- + building_satisfaction : dict + The ratio of satiesfied water for each building. + + """ self.wn = copy.deepcopy(self._clean_wn) # sets the damage state based on the current (change of the network) # and saves the current time - self.save_damage(damage, current_time) + self.save_damage(state, current_time) # prepare rewet inputs self.make_rewet_inputs(current_time, next_time) @@ -91,7 +139,7 @@ def system_performance(self, self.set_new_demand(state) # apply_damages - ## self.apply_damage() + # self.apply_damage() # run WDN performance evaluation self.run_performance(current_time, next_time) @@ -101,7 +149,8 @@ def system_performance(self, # Get result building_satisfaction = self.get_building_data_satisfaction( - method="mean") + method="mean" + ) self.building_satisfaction = building_satisfaction return building_satisfaction @@ -139,12 +188,12 @@ def read_inp_file(self, inp_file): if node_name not in self.nodes: self.nodes[node_name] = {} - self.nodes[node_name]["initial_demand"] = \ - node_demand_base_value + self.nodes[node_name][ + "initial_demand" + ] = node_demand_base_value self.nodes[node_name]["coordinates"] = node.coordinates - def set_asset_data(self, state): wdn_state = state["WaterDistributionNetwork"] wdn_state = wdn_state.get("Pipe", []) @@ -162,13 +211,15 @@ def set_asset_data(self, state): for element_key, element_data in sub_asset_type_data.items(): asset_id = element_data["GeneralInformation"]["AIM_id"] if asset_id != element_key: - raise ValueError("The rationality behidn the workdflow" - "is that oth aim-id and keys be the" - "same") + raise ValueError( + "The rationality behidn the workdflow" + "is that oth aim-id and keys be the" + "same" + ) - self.asset_information[asset_type]\ - [sub_asset_type]\ - [asset_id] = element_data["GeneralInformation"] + self.asset_information[asset_type][sub_asset_type][ + asset_id + ] = element_data["GeneralInformation"] building_state = state["Buildings"]["Building"] @@ -189,28 +240,29 @@ def set_asset_data(self, state): self.buildings[building_id] = cur_building - - def set_rewet_damage(self, damage, damage_time): + def update_state_with_damages(self, damage, damage_time, state): damage = damage["WaterDistributionNetwork"] pipe_damage = damage.get("Pipe", []) - damage_list = [] for asset_id, damage_location in pipe_damage.items(): damage_location_info = damage_location["Damage"] - aggregate_keys =[ - key for key in damage_location_info - if "aggregate-" in key] + aggregate_keys = [ + key for key in damage_location_info if "aggregate-" in key + ] aggregate_keys.sort() - aggregate_results = [damage_location_info[key] - for key in aggregate_keys] + aggregate_results = [ + damage_location_info[key] for key in aggregate_keys + ] - segment_sizes = len(aggregate_results ) + segment_sizes = len(aggregate_results) segment_step = 1 / segment_sizes c = 0 + cur_pipe_damage_location_list = [] + cur_pipe_damage_location_type = [] for damage_val in aggregate_results: if damage_val > 0: if damage_val == 1: @@ -218,25 +270,76 @@ def set_rewet_damage(self, damage, damage_time): elif damage_val == 2: damage_type = "break" else: - raise ValueError("The damage type must be eother " - "1 or 2") + raise ValueError( + "The damage type must be eother " "1 or 2" + ) else: continue - cur_loc = c * segment_step + segment_step / 2 + cur_pipe_damage_location_list.append(cur_loc) + cur_pipe_damage_location_type.append(damage_type) + + wdn_state = state["WaterDistributionNetwork"] + pipe_state = wdn_state.get("Pipe") + + if "Damage" in pipe_state[asset_id]: + raise ValueError( + f"Damage is already exist for Pipe " f"{asset_id}" + ) + + pipe_state[asset_id]["Damage"] = dict() + pipe_state[asset_id]["Damage"][ + "Location" + ] = cur_pipe_damage_location_list - c += 1 + pipe_state[asset_id]["Damage"][ + "Type" + ] = cur_pipe_damage_location_type + + def set_rewet_damage_from_state(self, state, damage_time): + state = state["WaterDistributionNetwork"] + pipe_damage = state.get("Pipe", []) + + damage_list = [] + for asset_id, pipe_info in pipe_damage.items(): + damage_location_info = pipe_info["Damage"] + damage_location_list = damage_location_info["Location"] + damage_type_list = damage_location_info["Type"] + + if len(damage_location_list) != len(damage_type_list): + raise ValueError( + "The size of types and locationis not the" " same." + ) + + segment_sizes = len(damage_location_list) + + # if segment_sizes == 0: + # continue + + pipe_id = self.asset_information["WaterDistributionNetwork"][ + "Pipe" + ][asset_id]["InpID"] + + for c in range(segment_sizes): + cur_loc = damage_location_list[c] + damage_type = damage_type_list[c] + + damage_list.append( + { + "pipe_id": pipe_id, + "damage_loc": cur_loc, + "type": damage_type, + "Material": "CI", + } + ) - pipe_id = self.asset_information["WaterDistributionNetwork"]\ - ["Pipe"][asset_id]["InpID"] - damage_list.append( {"pipe_id": pipe_id, "damage_loc": cur_loc, - "type": damage_type, "Material": "CI"} - ) damage_list.reverse() - self.pipe_damage = pd.Series(data=damage_list, - index=[damage_time for val in - damage_list], dtype="O") + self.pipe_damage = pd.Series( + data=damage_list, + index=[damage_time for val in damage_list], + dtype="O", + ) self.node_damage = pd.Series(dtype="O") @@ -244,12 +347,26 @@ def set_rewet_damage(self, damage, damage_time): self.tank_damage = pd.Series(dtype="O") - - def set_initial_water_damage_state(self, damage, damage_time): - self.damage_state[damage_time] = damage - self.set_rewet_damage(self.damage_state[damage_time], damage_time) + if damage_time in self.damage_state: + raise ValueError( + f"Time {damage_time} still exists in" f" damage state." + ) def couple_buildings_to_demand_nodes(self, state): + """ + Couple building to the demand nodes based on their coordinates. + + Parameters + ---------- + state :dict + State file content. + + Returns + ------- + None. + + """ + building_state = state["Buildings"]["Building"] building_id_list = [] @@ -260,17 +377,18 @@ def couple_buildings_to_demand_nodes(self, state): self.building_coordinates.append(coordinate) + demand_node_coordinate_list = [ + val["coordinates"] for key, val in self.nodes.items() + ] - demand_node_coordinate_list = [val["coordinates"] for key, val in - self.nodes.items()] - - demand_node_name_list = [key for key, val in - self.nodes.items()] + demand_node_name_list = [key for key, val in self.nodes.items()] - kmeans = KMeans(n_clusters=len(demand_node_coordinate_list), - init=demand_node_coordinate_list, - n_init=1, - random_state=0) + kmeans = KMeans( + n_clusters=len(demand_node_coordinate_list), + init=demand_node_coordinate_list, + n_init=1, + random_state=0, + ) kmeans.fit(self.building_coordinates) @@ -288,11 +406,12 @@ def couple_buildings_to_demand_nodes(self, state): building_id = building_id_list[building_l] self.demand_node_to_building[node_name].append(building_id) - for node_name in self.demand_node_to_building: building_name_list = self.demand_node_to_building[node_name] - population_list = [self.buildings[bldg_id]["initial_population"] - for bldg_id in building_name_list] + population_list = [ + self.buildings[bldg_id]["initial_population"] + for bldg_id in building_name_list + ] total_initial_population = sum(population_list) @@ -300,25 +419,30 @@ def couple_buildings_to_demand_nodes(self, state): initial_node_demand = cur_node["initial_demand"] if initial_node_demand == 0 and total_initial_population > 0: - Warning(f"Initial demand for node {node_name} is 0." + - "Thus, the demand ratio in buildidng(s)" + - "{repr(building_name_list).strip('[').strip(']'))}" + - "is naturally ineffective and their demand is not met." - ) + Warning( + f"Initial demand for node {node_name} is 0." + f" Thus, the demand ratio in buildidng(s)" + f" {repr(building_name_list).strip('[').strip(']')}" + f" is naturally ineffective and their demand" + f" is not met." + ) if total_initial_population == 0: - Warning(f"The population assigned to {node_name} is 0." + - "Thus, the demand ratio in buildidng(s)" + - "{repr(building_name_list).strip('[').strip(']'))}" + - "is ignored." - ) + Warning( + f"The population assigned to {node_name} is 0." + f" Thus, the demand ratio in buildidng(s)" + f" {repr(building_name_list).strip('[').strip(']')}" + f" is ignored." + ) + # We assume that population in State does not change in teh course # of recovery. Thus, the population is intiial population. For more # clarity, we name the variable "initial_population". # It does not mean that there will be ffdifferent population in # the course of recovery - self.nodes[node_name]["initial_population"] = \ - total_initial_population + self.nodes[node_name][ + "initial_population" + ] = total_initial_population self.nodes[node_name]["initial_node_demand"] = initial_node_demand @@ -326,15 +450,17 @@ def couple_buildings_to_demand_nodes(self, state): pop = self.buildings[bldg_id]["initial_population"] if total_initial_population != 0: - cur_bldg_initial_demand = \ + cur_bldg_initial_demand = ( pop / total_initial_population * initial_node_demand + ) else: cur_bldg_initial_demand = None - self.buildings[bldg_id]["initial_demand"] = \ - cur_bldg_initial_demand + self.buildings[bldg_id][ + "initial_demand" + ] = cur_bldg_initial_demand - def save_damage(self, damage, current_time): + def save_damage(self, state, current_time): """ Convert and save the dmaages that are set before @@ -366,8 +492,20 @@ def save_damage(self, damage, current_time): tank_path = str(tank_path) pump_path = str(pump_path) - self.damage_state[current_time] = damage - self.set_rewet_damage(self.damage_state[current_time], current_time) + # self.damage_state[current_time] = damage + self.set_rewet_damage_from_state(state, current_time) + + if current_time in self.damage_state: + raise ValueError( + f"The time {current_time} is already in " f" damage state." + ) + + self.damage_state[current_time] = { + "Pipe": self.pipe_damage, + "Node": self.node_damage, + "Pump": self.pump_damage, + "Tank": self.tank_damage, + } self.pipe_damage.to_pickle(pipe_path) self.node_damage.to_pickle(node_path) @@ -375,8 +513,8 @@ def save_damage(self, damage, current_time): self.tank_damage.to_pickle(tank_path) scn_postfix_list = random.choices( - ["A","B","C","D","E","F","G","H","I","J","L","M"], - k=0) + ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "L", "M"], k=0 + ) scn_postfix = "" for p in scn_postfix_list: @@ -394,7 +532,6 @@ def save_damage(self, damage, current_time): self.list_path = list_path def run_performance(self, current_time, next_time): - """ Runs the performance model using a hydraic solver (REWET) @@ -419,11 +556,14 @@ def run_performance(self, current_time, next_time): """ if current_time > next_time: - raise ValueError("Current tiime cannot be bigger than the next" - "time") + raise ValueError( + "Current tiime cannot be bigger than the next" "time" + ) if abs(next_time - current_time) % self.hydraulic_time_step != 0: - raise ValueError("next_time - current_time must be a factor of " - "the hydraulci time step.") + raise ValueError( + "next_time - current_time must be a factor of " + "the hydraulci time step." + ) status = self.rewet.initiate(current_time=current_time, debug=True) if status != 0: @@ -436,23 +576,24 @@ def run_performance(self, current_time, next_time): if status != 0: raise ValueError(f"There is an error: {status}") - return dict def make_rewet_inputs(self, current_time, next_time): - settings = rh.get_rewet_hydraulic_basic_setting() + settings = get_rewet_hydraulic_basic_setting() run_time = next_time - current_time list_file_path = Path(self.list_path) list_file_path = list_file_path.resolve() if not list_file_path.exists(): - raise ValueError(f"The list file does not exists: " - f"{str(list_file_path)}") + raise ValueError( + f"The list file does not exists: " f"{str(list_file_path)}" + ) list_file_path = str(list_file_path) - temp_dir = Path(TEMP_DIR).resolve() + temp_dir = Path(TEMP_DIR).resolve() if not temp_dir.exists(): - raise ValueError(f"The temp directory does not exists: " - f"{str(temp_dir)}") + raise ValueError( + f"The temp directory does not exists: " f"{str(temp_dir)}" + ) temp_dir = str(temp_dir) settings["RUN_TIME"] = run_time @@ -460,10 +601,10 @@ def make_rewet_inputs(self, current_time, next_time): settings["result_directory"] = RESULT_DIR settings["temp_directory"] = temp_dir settings["WN_INP"] = self.inp_file_path - settings['pipe_damage_file_list'] = list_file_path - settings['pipe_damage_file_directory'] = temp_dir - settings['Restoration_on'] = False - settings['Pipe_damage_input_method'] = "pickle" + settings["pipe_damage_file_list"] = list_file_path + settings["pipe_damage_file_directory"] = temp_dir + settings["Restoration_on"] = False + settings["Pipe_damage_input_method"] = "pickle" input_file_path = Path(INPUT_FILE_DIR) / "rewet_input.json" input_file_path = input_file_path.resolve() @@ -490,24 +631,24 @@ def get_building_data_satisfaction(self, method): for node_name in self.demand_node_to_building: node_demand_ratio = demand_sat[node_name] building_names = self.demand_node_to_building[node_name] - cur_satisified_demand_building = dict(zip( - building_names, - [node_demand_ratio]*len(building_names))) + cur_satisified_demand_building = dict( + zip(building_names, [node_demand_ratio] * len(building_names)) + ) building_demand_satisfaction_ratio.update( - cur_satisified_demand_building) + cur_satisified_demand_building + ) return building_demand_satisfaction_ratio - - def set_new_demand(self, state): for node_name in self.demand_node_to_building: - cur_node = self.nodes[node_name] - total_initial_population = \ - self.nodes[node_name]["initial_population"] + # cur_node = self.nodes[node_name] + total_initial_population = self.nodes[node_name][ + "initial_population" + ] if not total_initial_population > 0: continue @@ -520,17 +661,19 @@ def set_new_demand(self, state): for bldg_id in building_name_list: - cur_bldg_initial_demand = \ - self.buildings[bldg_id]["initial_demand"] + cur_bldg_initial_demand = self.buildings[bldg_id][ + "initial_demand" + ] - cur_bldg_deamnd_ratio = \ - building[bldg_id]["GeneralInformation"]["Population_Ratio"] + cur_bldg_deamnd_ratio = building[bldg_id][ + "GeneralInformation" + ]["Population_Ratio"] - cur_bldg_new_deamnd = \ + cur_bldg_new_deamnd = ( cur_bldg_deamnd_ratio * cur_bldg_initial_demand + ) - self.buildings[bldg_id]["current_demand"] = \ - cur_bldg_new_deamnd + self.buildings[bldg_id]["current_demand"] = cur_bldg_new_deamnd node_new_demand += cur_bldg_new_deamnd @@ -539,8 +682,15 @@ def set_new_demand(self, state): node.demand_timeseries_list[0].base_value = node_new_demand +def get_rewet_hydraulic_basic_setting(): + settings = rewet.Input.Settings.Settings() + settings_dict = settings.process.settings + # settings_dict.update(settings.scenario.settings) + + return settings_dict -if __name__ == '__main__': + +if __name__ == "__main__": # raise RuntimeError("This file should not be run") with open("Results_det.json", "rt") as f: state = json.load(f) @@ -552,14 +702,9 @@ def set_new_demand(self, state): interface = REWETPyReCoDes() interface.system_state(state, damage, inp_file) - result = interface.system_performance(state, - damage, - 0, - 24*3600) + result = interface.system_performance(state, damage, 0, 24 * 3600) # = interface.system_performance(state, - #damage, - #1*24*3600, - #2*24*3600) - - + # damage, + # 1*24*3600, + # 2*24*3600) diff --git a/modules/performREC/pyrecodes/damage_convertor.py b/modules/performREC/pyrecodes/damage_convertor.py index d460bbbcb..3597f42ae 100644 --- a/modules/performREC/pyrecodes/damage_convertor.py +++ b/modules/performREC/pyrecodes/damage_convertor.py @@ -1,9 +1,39 @@ -# -*- coding: utf-8 -*- -""" -Created on Mon Jan 8 09:40:15 2024 - -@author: naeim -""" +# Copyright (c) 2024 The Regents of the University of California +# Copyright (c) 2024 Leland Stanford Junior University +# +# This file is part of whale. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# You should have received a copy of the BSD 3-Clause License along with +# whale. If not, see . +# +# Contributors: +# Sina Naeimi import os from pathlib import Path @@ -13,9 +43,9 @@ CBIG_int = int(1e9) - - -def createPipeDamageInputForREWET(pipe_damage_data, run_dir, event_time, sc_geojson): +def createPipeDamageInputForREWET( + pipe_damage_data, run_dir, event_time, sc_geojson +): """ Creates REWET-style piep damage file. @@ -38,36 +68,47 @@ def createPipeDamageInputForREWET(pipe_damage_data, run_dir, event_time, sc_geoj """ pipe_id_list = [key for key in pipe_damage_data] - + damage_list = [] damage_time = event_time sc_geojson_file = preprocessorIO.readJSONFile(sc_geojson) - pipe_data = [ss for ss in sc_geojson_file["features"] if ss["properties"]["type"]=="Pipe"] + pipe_data = [ + ss + for ss in sc_geojson_file["features"] + if ss["properties"]["type"] == "Pipe" + ] pipe_index = [str(ss["id"]) for ss in pipe_data] pipe_id = [ss["properties"]["InpID"] for ss in pipe_data] pipe_index_to_id = dict(zip(pipe_index, pipe_id)) - + for pipe_id in pipe_id_list: cur_data = pipe_damage_data[pipe_id] cur_damage = cur_data["Damage"] - cur_demand = cur_data["Demand"] - - aim_data = findAndReadAIMFile(pipe_id,os.path.join( - "Results", "WaterDistributionNetwork", "Pipe"), run_dir) - + # cur_demand = cur_data["Demand"] + + aim_data = findAndReadAIMFile( + pipe_id, + os.path.join("Results", "WaterDistributionNetwork", "Pipe"), + run_dir, + ) + material = aim_data["GeneralInformation"].get("Material", None) - - if material == None: - #raise ValueError("Material is none") + + if material is None: + # raise ValueError("Material is none") material = "CI" - - aggregates_list = [cur_agg for cur_agg in list( cur_damage.keys() ) if "aggregate" in cur_agg] - segment_sizes = len(aggregates_list ) + + aggregates_list = [ + cur_agg + for cur_agg in list(cur_damage.keys()) + if "aggregate" in cur_agg + ] + segment_sizes = len(aggregates_list) segment_step = 1 / segment_sizes c = 0 - - for cur_agg in aggregates_list: #cur_damage["aggregate"]: + + for cur_agg in aggregates_list: # cur_damage["aggregate"]: damage_val = cur_damage[cur_agg] if damage_val > 0: if damage_val == 1: @@ -78,24 +119,29 @@ def createPipeDamageInputForREWET(pipe_damage_data, run_dir, event_time, sc_geoj raise ValueError("The damage type must be eother 1 or 2") else: continue - cur_loc = c * segment_step + segment_step / 2 - #print(cur_loc) + # print(cur_loc) c += 1 - damage_list.append( {"pipe_id": pipe_index_to_id[pipe_id], "damage_loc": cur_loc, - "type": damage_type, "Material": material} - ) + damage_list.append( + { + "pipe_id": pipe_index_to_id[pipe_id], + "damage_loc": cur_loc, + "type": damage_type, + "Material": material, + } + ) damage_list.reverse() - pipe_damage_list = pd.Series(data=damage_list, - index=[damage_time for val in damage_list], dtype="O") - - #REWET_input_data["Pipe_damage_list"] = pipe_damage_list - #REWET_input_data["AIM"] = aim_data - - + pipe_damage_list = pd.Series( + data=damage_list, index=[damage_time for val in damage_list], dtype="O" + ) + + # REWET_input_data["Pipe_damage_list"] = pipe_damage_list + # REWET_input_data["AIM"] = aim_data + return pipe_damage_list + def createNodeDamageInputForREWET(node_damage_data, run_dir, event_time): """ Creates REWET-style node damage file. @@ -114,39 +160,50 @@ def createNodeDamageInputForREWET(node_damage_data, run_dir, event_time): """ node_id_list = [key for key in node_damage_data] - + damage_list = [] damage_time = event_time - + for node_id in node_id_list: cur_damage = node_damage_data[node_id] - aggregates_list = [cur_agg for cur_agg in list( cur_damage.keys() ) if "aggregate" in cur_agg] - + aggregates_list = [ + cur_agg + for cur_agg in list(cur_damage.keys()) + if "aggregate" in cur_agg + ] + if len(aggregates_list) == 0: continue - + cur_data = node_damage_data[node_id] cur_damage = cur_data["Damage"] - cur_demand = cur_data["Demand"] - - aim_data = findAndReadAIMFile(node_id,os.path.join( - "Results", "WaterDistributionNetwork", "Node"), - run_dir) - + # cur_demand = cur_data["Demand"] + + aim_data = findAndReadAIMFile( + node_id, + os.path.join("Results", "WaterDistributionNetwork", "Node"), + run_dir, + ) + total_length = aim_data["GeneralInformation"].get("Total_length", None) total_number_of_damages = cur_damage["aggregate"] - - damage_list.append( {"node_name": node_id, - "number_of_damages": total_number_of_damages, - "node_Pipe_Length": total_length} - ) - - node_damage_list = pd.Series(data=damage_list, - index=[damage_time for val in damage_list], dtype="O") - + + damage_list.append( + { + "node_name": node_id, + "number_of_damages": total_number_of_damages, + "node_Pipe_Length": total_length, + } + ) + + node_damage_list = pd.Series( + data=damage_list, index=[damage_time for val in damage_list], dtype="O" + ) + return node_damage_list - + + def createPumpDamageInputForREWET(pump_damage_data, REWET_input_data): """ Creates REWET-style pump damage file. @@ -165,36 +222,41 @@ def createPumpDamageInputForREWET(pump_damage_data, REWET_input_data): """ pump_id_list = [key for key in pump_damage_data] - + damage_list = [] damage_time = REWET_input_data["event_time"] - + for pump_id in pump_id_list: cur_data = pump_damage_data[pump_id] cur_damage = cur_data["Damage"] cur_repair_time = cur_data["Repair"] - + if cur_damage == 0: - continue # cur_damage_state = 0 means undamaged pump - + continue # cur_damage_state = 0 means undamaged pump + # I'm not sure if we need any data about the pump at this point - - #aim_data = findAndReadAIMFile(tank_id, os.path.join( - #"Results", "WaterDistributionNetwork", "Pump"), - #REWET_input_data["run_dir"]) - - #We are getting this data from PELICUN - #restore_time = getPumpRetsoreTime(cur_damage) - damage_list.append( {"pump_id": pump_id, - "time": damage_time, "Restore_time": cur_repair_time} - ) - pump_damage_list = pd.Series(index=[damage_time for val in damage_list], data=damage_list) - + + # aim_data = findAndReadAIMFile(tank_id, os.path.join( + # "Results", "WaterDistributionNetwork", "Pump"), + # REWET_input_data["run_dir"]) + + # We are getting this data from PELICUN + # restore_time = getPumpRetsoreTime(cur_damage) + damage_list.append( + { + "pump_id": pump_id, + "time": damage_time, + "Restore_time": cur_repair_time, + } + ) + pump_damage_list = pd.Series( + index=[damage_time for val in damage_list], data=damage_list + ) + return pump_damage_list - - - + + def createTankDamageInputForREWET(tank_damage_data, REWET_input_data): """ Creates REWET-style Tank damage file. @@ -212,38 +274,44 @@ def createTankDamageInputForREWET(tank_damage_data, REWET_input_data): REWET-style tank damage file. """ tank_id_list = [key for key in tank_damage_data] - + damage_list = [] damage_time = REWET_input_data["event_time"] - + for tank_id in tank_id_list: cur_data = tank_damage_data[tank_id] cur_damage = cur_data["Damage"] cur_repair_time = cur_data["Repair"] - + if cur_damage == 0: - continue # cur_damage_state = 0 meeans undamged tank - -# ============================================================================= -# # We are getting his data from REWET -# -# aim_data = findAndReadAIMFile(tank_id, os.path.join( -# "Results", "WaterDistributionNetwork", "Tank"), -# REWET_input_data["run_dir"]) -# tank_type = aim_data["GeneralInformation"].get("Type", None) -# restore_time = getTankRetsoreTime(tank_type, cur_damage) -# ============================================================================= - - damage_list.append( {"tank_id": tank_id, - "time": damage_time, "Restore_time": cur_repair_time} - ) - - tank_damage_list = pd.Series(index=[damage_time for val in damage_list], data=damage_list) - + continue # cur_damage_state = 0 meeans undamged tank + + # ============================================================================= + # # We are getting his data from REWET + # + # aim_data = findAndReadAIMFile(tank_id, os.path.join( + # "Results", "WaterDistributionNetwork", "Tank"), + # REWET_input_data["run_dir"]) + # tank_type = aim_data["GeneralInformation"].get("Type", None) + # restore_time = getTankRetsoreTime(tank_type, cur_damage) + # ============================================================================= + + damage_list.append( + { + "tank_id": tank_id, + "time": damage_time, + "Restore_time": cur_repair_time, + } + ) + + tank_damage_list = pd.Series( + index=[damage_time for val in damage_list], data=damage_list + ) + return tank_damage_list - - + + def findAndReadAIMFile(asset_id, asset_type, run_dir): """ Finds and read the AIM file for an asset. @@ -264,20 +332,27 @@ def findAndReadAIMFile(asset_id, asset_type, run_dir): """ - file_path = Path(run_dir, asset_type, str(asset_id), "templatedir", f"{asset_id}-AIM.json") - aim_file_data = preprocessorIO.readJSONFile(str(file_path) ) + file_path = Path( + run_dir, + asset_type, + str(asset_id), + "templatedir", + f"{asset_id}-AIM.json", + ) + aim_file_data = preprocessorIO.readJSONFile(str(file_path)) return aim_file_data - + + def getPumpRetsoreTime(damage_state): """ NOT USED! WE WILL GET IT FROM PELICUN - + Provides the restore time based on HAZUS repair time or any other approach available in the future. If damage state is slight, the restore time is 3 days (in seconds). If damage state is 2, the restore time is 7 days (in seconds). If damage state is 3 or 4, the restore time is indefinite (a big number). - + Parameters ---------- damage_state : Int @@ -287,29 +362,30 @@ def getPumpRetsoreTime(damage_state): Returns ------- Retstor time : int - + """ - + if damage_state == 1: restore_time = int(3 * 24 * 3600) elif damage_state == 2: restore_time = int(7 * 24 * 3600) else: restore_time = CBIG_int - + return restore_time + def getTankRetsoreTime(tank_type, damage_state): """ NOT USED! WE WILL GET IT FROM PELICUN - + Provides the restore time based on HAZUS repair time or any other approach available in the future. if damage state is slight, the restore time is 3 days (in seconds). If damage state is 2, the restore time is 7 days (in seconds). If damage state is 3 or 4, the restore time is indefinite (a big number). - + Parameters ---------- tank_type : STR @@ -321,19 +397,20 @@ def getTankRetsoreTime(tank_type, damage_state): Returns ------- Retstor time : int - + """ - + if damage_state == 1: restore_time = int(3 * 24 * 3600) elif damage_state == 2: restore_time = int(7 * 24 * 3600) else: restore_time = CBIG_int - + return restore_time + def readDamagefile(file_addr, run_dir, event_time, sc_geojson): """ Reads PELICUN damage files and create REWET-Style damage for all @@ -355,41 +432,45 @@ def readDamagefile(file_addr, run_dir, event_time, sc_geojson): """ # TODO: Make reading once for each scneario - - #wn = wntrfr.network.WaterNetworkModel(REWET_input_data["inp_file"] ) - + + # wn = wntrfr.network.WaterNetworkModel(REWET_input_data["inp_file"] ) + damage_data = preprocessorIO.readJSONFile(file_addr) - + wn_damage_data = damage_data["WaterDistributionNetwork"] if "Pipe" in wn_damage_data: pipe_damage_data = createPipeDamageInputForREWET( - wn_damage_data["Pipe"], run_dir, event_time, sc_geojson) + wn_damage_data["Pipe"], run_dir, event_time, sc_geojson + ) else: pipe_damage_data = pd.Series(dtype="O") - + if "Tank" in wn_damage_data: tank_damage_data = createTankDamageInputForREWET( - wn_damage_data["Tank"], run_dir, event_time) + wn_damage_data["Tank"], run_dir, event_time + ) else: tank_damage_data = pd.Series(dtype="O") - - if "Pump" in wn_damage_data: + + if "Pump" in wn_damage_data: pump_damage_data = createPumpDamageInputForREWET( - wn_damage_data["Pump"], run_dir, event_time) + wn_damage_data["Pump"], run_dir, event_time + ) else: pump_damage_data = pd.Series(dtype="O") - - if "Junction" in wn_damage_data: + + if "Junction" in wn_damage_data: node_damage_data = createNodeDamageInputForREWET( - wn_damage_data["Junction"], run_dir, event_time) + wn_damage_data["Junction"], run_dir, event_time + ) else: node_damage_data = pd.Series(dtype="O") - + damage_data = {} damage_data["Pipe"] = pipe_damage_data damage_data["Tank"] = tank_damage_data damage_data["Pump"] = pump_damage_data damage_data["Node"] = node_damage_data - - return damage_data \ No newline at end of file + + return damage_data From b6fe92387576f5b652dd009ada550f83e8a8286e Mon Sep 17 00:00:00 2001 From: Sina Naeimi Date: Tue, 27 Aug 2024 16:10:53 -0700 Subject: [PATCH 14/26] Hopefully Ruff check passes --- .../pyrecodes/PyReCode-REWET-preprocessor.py | 152 +++++++++++++++--- 1 file changed, 130 insertions(+), 22 deletions(-) diff --git a/modules/performREC/pyrecodes/PyReCode-REWET-preprocessor.py b/modules/performREC/pyrecodes/PyReCode-REWET-preprocessor.py index 6a097efdb..6a8ba235f 100644 --- a/modules/performREC/pyrecodes/PyReCode-REWET-preprocessor.py +++ b/modules/performREC/pyrecodes/PyReCode-REWET-preprocessor.py @@ -35,6 +35,8 @@ # Contributors: # Sina Naeimi +"""Provide Interface between REWET and PYReCoDes.""" + import random import json import copy @@ -51,6 +53,8 @@ class REWETPyReCoDes: + """Provdie the wrapper for REWET API.""" + def __init__(self): self.wn = None self._clean_wn = None @@ -71,7 +75,7 @@ def __init__(self): def system_state(self, state, damage, inp_file, damage_time=0): """ - Sets the WDN system for PyReCoDes Interface. + Set the WDN system for PyReCoDes Interface. Parameters ---------- @@ -90,7 +94,6 @@ def system_state(self, state, damage, inp_file, damage_time=0): None. """ - # read the inp file self.read_inp_file(inp_file) @@ -104,7 +107,7 @@ def system_state(self, state, damage, inp_file, damage_time=0): def system_performance(self, state, damage, current_time, next_time): """ - Assesses the system functionality. + Assess the system functionality. Parameters ---------- @@ -123,7 +126,6 @@ def system_performance(self, state, damage, current_time, next_time): The ratio of satiesfied water for each building. """ - self.wn = copy.deepcopy(self._clean_wn) # sets the damage state based on the current (change of the network) # and saves the current time @@ -157,7 +159,7 @@ def system_performance(self, state, damage, current_time, next_time): def read_inp_file(self, inp_file): """ - Reads the inp file + Read the inp file. Parameters ---------- @@ -169,7 +171,6 @@ def read_inp_file(self, inp_file): None. """ - self.inp_file_path = Path(inp_file) self.inp_file_path = self.inp_file_path.resolve() @@ -195,6 +196,24 @@ def read_inp_file(self, inp_file): self.nodes[node_name]["coordinates"] = node.coordinates def set_asset_data(self, state): + """ + Set the asset information from state file. + + Parameters + ---------- + state : dict + _det file. + + Raises + ------ + ValueError + Unexpecyed values exists in state file. + + Returns + ------- + None. + + """ wdn_state = state["WaterDistributionNetwork"] wdn_state = wdn_state.get("Pipe", []) @@ -212,7 +231,7 @@ def set_asset_data(self, state): asset_id = element_data["GeneralInformation"]["AIM_id"] if asset_id != element_key: raise ValueError( - "The rationality behidn the workdflow" + "The rationality behidd the workdflow" "is that oth aim-id and keys be the" "same" ) @@ -241,6 +260,28 @@ def set_asset_data(self, state): self.buildings[building_id] = cur_building def update_state_with_damages(self, damage, damage_time, state): + """ + Update the state dic with damages. + + Parameters + ---------- + damage : dict + _i file in dict form.. + damage_time : int + Damaeg time. + state : dict + _det file. + + Raises + ------ + ValueError + Unexpetced damage state in damage data. + + Returns + ------- + None. + + """ damage = damage["WaterDistributionNetwork"] pipe_damage = damage.get("Pipe", []) @@ -271,7 +312,7 @@ def update_state_with_damages(self, damage, damage_time, state): damage_type = "break" else: raise ValueError( - "The damage type must be eother " "1 or 2" + "The damage type must be either 1 or 2" ) else: continue @@ -298,6 +339,26 @@ def update_state_with_damages(self, damage, damage_time, state): ] = cur_pipe_damage_location_type def set_rewet_damage_from_state(self, state, damage_time): + """ + Set REWET damafe data from state at each tiem step. + + Parameters + ---------- + state : dict + _det file in dict format. + damage_time : int + Current damage time. + + Raises + ------ + ValueError + Unexpected or abnormal data in state. + + Returns + ------- + None. + + """ state = state["WaterDistributionNetwork"] pipe_damage = state.get("Pipe", []) @@ -314,9 +375,6 @@ def set_rewet_damage_from_state(self, state, damage_time): segment_sizes = len(damage_location_list) - # if segment_sizes == 0: - # continue - pipe_id = self.asset_information["WaterDistributionNetwork"][ "Pipe" ][asset_id]["InpID"] @@ -366,7 +424,6 @@ def couple_buildings_to_demand_nodes(self, state): None. """ - building_state = state["Buildings"]["Building"] building_id_list = [] @@ -462,7 +519,7 @@ def couple_buildings_to_demand_nodes(self, state): def save_damage(self, state, current_time): """ - Convert and save the dmaages that are set before + Convert and save the dmaages that are set before. Parameters ---------- @@ -533,7 +590,7 @@ def save_damage(self, state, current_time): def run_performance(self, current_time, next_time): """ - Runs the performance model using a hydraic solver (REWET) + Run the performance model using a hydraic solver (REWET). Parameters ---------- @@ -554,7 +611,6 @@ def run_performance(self, current_time, next_time): building. """ - if current_time > next_time: raise ValueError( "Current tiime cannot be bigger than the next" "time" @@ -579,6 +635,26 @@ def run_performance(self, current_time, next_time): return dict def make_rewet_inputs(self, current_time, next_time): + """ + Create setting input for REWET. + + Parameters + ---------- + current_time : int + Current time in seconds. + next_time : TYPE + Next stop time in seconds. + + Raises + ------ + ValueError + Path are not available. + + Returns + ------- + None. + + """ settings = get_rewet_hydraulic_basic_setting() run_time = next_time - current_time list_file_path = Path(self.list_path) @@ -614,7 +690,25 @@ def make_rewet_inputs(self, current_time, next_time): self.input_file_path = str(input_file_path) def get_building_data_satisfaction(self, method): + """ + Get building water satiesfaction data. + Parameters + ---------- + method : str + MEAB, MAX, or MIN. + + Raises + ------ + ValueError + Unrecognizable method is given. + + Returns + ------- + building_demand_satisfaction_ratio : dict + Building satiesfied ratio. + + """ demand_sat = self.rewet.get_satisfied_demand_ratio() if method.upper() == "MEAN": @@ -642,7 +736,19 @@ def get_building_data_satisfaction(self, method): return building_demand_satisfaction_ratio def set_new_demand(self, state): + """ + Set new demand from state. + Parameters + ---------- + state : dict + _det file in dict format. + + Returns + ------- + None. + + """ for node_name in self.demand_node_to_building: # cur_node = self.nodes[node_name] @@ -683,15 +789,22 @@ def set_new_demand(self, state): def get_rewet_hydraulic_basic_setting(): + """ + Create basic settings for rewet's input. + + Returns + ------- + settings_dict : dict + REWET input. + + """ settings = rewet.Input.Settings.Settings() settings_dict = settings.process.settings - # settings_dict.update(settings.scenario.settings) return settings_dict if __name__ == "__main__": - # raise RuntimeError("This file should not be run") with open("Results_det.json", "rt") as f: state = json.load(f) @@ -703,8 +816,3 @@ def get_rewet_hydraulic_basic_setting(): interface = REWETPyReCoDes() interface.system_state(state, damage, inp_file) result = interface.system_performance(state, damage, 0, 24 * 3600) - - # = interface.system_performance(state, - # damage, - # 1*24*3600, - # 2*24*3600) From 958c4c6ba03dc18cf8dec0426fb4249cdfe063ea Mon Sep 17 00:00:00 2001 From: Sina Naeimi Date: Tue, 27 Aug 2024 16:28:44 -0700 Subject: [PATCH 15/26] Ruff Check2 --- .../pyrecodes/PyReCode-REWET-preprocessor.py | 16 ++--- .../performREC/pyrecodes/damage_convertor.py | 59 +++++++++---------- 2 files changed, 37 insertions(+), 38 deletions(-) diff --git a/modules/performREC/pyrecodes/PyReCode-REWET-preprocessor.py b/modules/performREC/pyrecodes/PyReCode-REWET-preprocessor.py index 6a8ba235f..fd2489653 100644 --- a/modules/performREC/pyrecodes/PyReCode-REWET-preprocessor.py +++ b/modules/performREC/pyrecodes/PyReCode-REWET-preprocessor.py @@ -53,7 +53,7 @@ class REWETPyReCoDes: - """Provdie the wrapper for REWET API.""" + """Provide the wrapper for REWET API.""" def __init__(self): self.wn = None @@ -80,7 +80,7 @@ def system_state(self, state, damage, inp_file, damage_time=0): Parameters ---------- state : dict - The State of system which is defiend in the R2DTool's system_det + The State of system which is defined in the R2DTool's system_det style. damage : dict The damage state in 2DTool's system_i style. @@ -179,7 +179,7 @@ def read_inp_file(self, inp_file): self.inp_file_path = str(self.inp_file_path) - # read the inp file and create the WDN obejct file + # Read the inp file and create the WDN object file self.wn = wntrfr.network.model.WaterNetworkModel(self.inp_file_path) self._clean_wn = copy.deepcopy(self.wn) @@ -275,7 +275,7 @@ def update_state_with_damages(self, damage, damage_time, state): Raises ------ ValueError - Unexpetced damage state in damage data. + Unexpected damage state in damage data. Returns ------- @@ -340,7 +340,7 @@ def update_state_with_damages(self, damage, damage_time, state): def set_rewet_damage_from_state(self, state, damage_time): """ - Set REWET damafe data from state at each tiem step. + Set REWET damafe data from state at each time step. Parameters ---------- @@ -492,9 +492,9 @@ def couple_buildings_to_demand_nodes(self, state): f" is ignored." ) - # We assume that population in State does not change in teh course - # of recovery. Thus, the population is intiial population. For more - # clarity, we name the variable "initial_population". + # We assume that population in the State does not change in the + # course of recovery. Thus, the population is initial population. + # For more clarity, we name the variable "initial_population". # It does not mean that there will be ffdifferent population in # the course of recovery self.nodes[node_name][ diff --git a/modules/performREC/pyrecodes/damage_convertor.py b/modules/performREC/pyrecodes/damage_convertor.py index 3597f42ae..a3aa1aa57 100644 --- a/modules/performREC/pyrecodes/damage_convertor.py +++ b/modules/performREC/pyrecodes/damage_convertor.py @@ -35,6 +35,8 @@ # Contributors: # Sina Naeimi +"""Converts damages from Pelicun to REWET format.""" + import os from pathlib import Path import pandas as pd @@ -47,14 +49,18 @@ def createPipeDamageInputForREWET( pipe_damage_data, run_dir, event_time, sc_geojson ): """ - Creates REWET-style piep damage file. + Create REWET-style piep damage file. Parameters ---------- pipe_damage_data : dict Pipe damage data from PELICUN. - REWET_input_data : dict - REWET input data. + run_dir : path + Run dierctory. + event_time : int + Damage time. + sc_geojson : dict + Backend sc_geojson. Raises ------ @@ -144,14 +150,16 @@ def createPipeDamageInputForREWET( def createNodeDamageInputForREWET(node_damage_data, run_dir, event_time): """ - Creates REWET-style node damage file. + Create REWET-style node damage file. Parameters ---------- node_damage_data : dict Node damage data from PELICUN. - REWET_input_data : dict - REWET input data. + run_dir : path + Run dierctory. + event_time : int + Damage time. Returns ------- @@ -206,7 +214,7 @@ def createNodeDamageInputForREWET(node_damage_data, run_dir, event_time): def createPumpDamageInputForREWET(pump_damage_data, REWET_input_data): """ - Creates REWET-style pump damage file. + Create REWET-style pump damage file. Parameters ---------- @@ -259,7 +267,7 @@ def createPumpDamageInputForREWET(pump_damage_data, REWET_input_data): def createTankDamageInputForREWET(tank_damage_data, REWET_input_data): """ - Creates REWET-style Tank damage file. + Create REWET-style Tank damage file. Parameters ---------- @@ -287,16 +295,6 @@ def createTankDamageInputForREWET(tank_damage_data, REWET_input_data): if cur_damage == 0: continue # cur_damage_state = 0 meeans undamged tank - # ============================================================================= - # # We are getting his data from REWET - # - # aim_data = findAndReadAIMFile(tank_id, os.path.join( - # "Results", "WaterDistributionNetwork", "Tank"), - # REWET_input_data["run_dir"]) - # tank_type = aim_data["GeneralInformation"].get("Type", None) - # restore_time = getTankRetsoreTime(tank_type, cur_damage) - # ============================================================================= - damage_list.append( { "tank_id": tank_id, @@ -314,7 +312,7 @@ def createTankDamageInputForREWET(tank_damage_data, REWET_input_data): def findAndReadAIMFile(asset_id, asset_type, run_dir): """ - Finds and read the AIM file for an asset. + Find and read the AIM file for an asset. Parameters ---------- @@ -331,7 +329,6 @@ def findAndReadAIMFile(asset_id, asset_type, run_dir): AIM file data as a dict. """ - file_path = Path( run_dir, asset_type, @@ -343,11 +340,12 @@ def findAndReadAIMFile(asset_id, asset_type, run_dir): return aim_file_data +# Not UsedWE WILL GET IT FROM PELICUN def getPumpRetsoreTime(damage_state): """ - NOT USED! WE WILL GET IT FROM PELICUN + Provide the restore time. - Provides the restore time based on HAZUS repair time or any other + Based on HAZUS repair time or any other approach available in the future. If damage state is slight, the restore time is 3 days (in seconds). If damage state is 2, the restore time is 7 days (in seconds). If damage state is 3 or 4, the restore time is @@ -365,7 +363,6 @@ def getPumpRetsoreTime(damage_state): """ - if damage_state == 1: restore_time = int(3 * 24 * 3600) elif damage_state == 2: @@ -376,11 +373,12 @@ def getPumpRetsoreTime(damage_state): return restore_time +# NOT USED! WE WILL GET IT FROM PELICUN def getTankRetsoreTime(tank_type, damage_state): """ - NOT USED! WE WILL GET IT FROM PELICUN + Provide the restore time. - Provides the restore time based on HAZUS repair time or any other + Based on HAZUS repair time or any other approach available in the future. if damage state is slight, the restore time is 3 days (in seconds). If damage state is 2, the restore time is 7 days (in seconds). If damage state is 3 or 4, the restore time is @@ -389,7 +387,7 @@ def getTankRetsoreTime(tank_type, damage_state): Parameters ---------- tank_type : STR - Tank type based on the data schema. The parametr is not used for now. + Tank type based on the data schema. The parameter is not used for now. damage_state : Int Specifies the damage state (1 for slightly damages, 2 for moderate, 3 etensive, and 4 complete. @@ -400,7 +398,6 @@ def getTankRetsoreTime(tank_type, damage_state): """ - if damage_state == 1: restore_time = int(3 * 24 * 3600) elif damage_state == 2: @@ -413,7 +410,9 @@ def getTankRetsoreTime(tank_type, damage_state): def readDamagefile(file_addr, run_dir, event_time, sc_geojson): """ - Reads PELICUN damage files and create REWET-Style damage for all + Read PELICUN damage files. and create REWET-Style damage. + + Reads PELICUN damage files. and create REWET-Style damage for all WaterDistributionNetwork elements Parameters @@ -421,7 +420,7 @@ def readDamagefile(file_addr, run_dir, event_time, sc_geojson): file_addr : path PELICUN damage file in JSON format. REWET_input_data : dict - REWET input data, whcih is updated in the function. + REWET input data, which is updated in the function. scn_number : dict JSON FILE. @@ -431,7 +430,7 @@ def readDamagefile(file_addr, run_dir, event_time, sc_geojson): Damage data in PELICUN dict format. """ - # TODO: Make reading once for each scneario + # TODO(Sina): Make reading once for each scenario # wn = wntrfr.network.WaterNetworkModel(REWET_input_data["inp_file"] ) From 36f5f9cd4da72fd8f2e78d69bb02e95dfc6615cc Mon Sep 17 00:00:00 2001 From: Sina Naeimi Date: Tue, 27 Aug 2024 16:55:56 -0700 Subject: [PATCH 16/26] Ruff Check3 --- .../pyrecodes/PyReCode-REWET-preprocessor.py | 313 ++++++++---------- .../performREC/pyrecodes/damage_convertor.py | 163 +++++---- 2 files changed, 219 insertions(+), 257 deletions(-) diff --git a/modules/performREC/pyrecodes/PyReCode-REWET-preprocessor.py b/modules/performREC/pyrecodes/PyReCode-REWET-preprocessor.py index fd2489653..c951ec51a 100644 --- a/modules/performREC/pyrecodes/PyReCode-REWET-preprocessor.py +++ b/modules/performREC/pyrecodes/PyReCode-REWET-preprocessor.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024 The Regents of the University of California +# Copyright (c) 2024 The Regents of the University of California # noqa: INP001 # Copyright (c) 2024 Leland Stanford Junior University # # This file is part of whale. @@ -37,19 +37,20 @@ """Provide Interface between REWET and PYReCoDes.""" -import random -import json import copy -import rewet +import json +import random from pathlib import Path -import wntrfr + import pandas as pd -from sklearn.cluster import KMeans +import rewet +import wntrfr from rewet.api import API +from sklearn.cluster import KMeans -TEMP_DIR = "./" -RESULT_DIR = "./rewet_result" -INPUT_FILE_DIR = "./" +TEMP_DIR = './' +RESULT_DIR = './rewet_result' +INPUT_FILE_DIR = './' class REWETPyReCoDes: @@ -105,7 +106,7 @@ def system_state(self, state, damage, inp_file, damage_time=0): # couple building to the demand nodes self.couple_buildings_to_demand_nodes(state) - def system_performance(self, state, damage, current_time, next_time): + def system_performance(self, state, current_time, next_time): """ Assess the system functionality. @@ -150,9 +151,7 @@ def system_performance(self, state, damage, current_time, next_time): self.rewet.save_result() # Get result - building_satisfaction = self.get_building_data_satisfaction( - method="mean" - ) + building_satisfaction = self.get_building_data_satisfaction(method='mean') self.building_satisfaction = building_satisfaction return building_satisfaction @@ -175,7 +174,7 @@ def read_inp_file(self, inp_file): self.inp_file_path = self.inp_file_path.resolve() if not self.inp_file_path.exists(): - raise ValueError(f"There inp file does not exists: {inp_file}") + raise ValueError(f'There inp file does not exists: {inp_file}') # noqa: EM102, TRY003 self.inp_file_path = str(self.inp_file_path) @@ -189,11 +188,9 @@ def read_inp_file(self, inp_file): if node_name not in self.nodes: self.nodes[node_name] = {} - self.nodes[node_name][ - "initial_demand" - ] = node_demand_base_value + self.nodes[node_name]['initial_demand'] = node_demand_base_value - self.nodes[node_name]["coordinates"] = node.coordinates + self.nodes[node_name]['coordinates'] = node.coordinates def set_asset_data(self, state): """ @@ -214,8 +211,8 @@ def set_asset_data(self, state): None. """ - wdn_state = state["WaterDistributionNetwork"] - wdn_state = wdn_state.get("Pipe", []) + wdn_state = state['WaterDistributionNetwork'] + wdn_state = wdn_state.get('Pipe', []) for asset_type, asset_type_data in state.items(): if asset_type not in self.asset_information: @@ -228,23 +225,23 @@ def set_asset_data(self, state): self.asset_information[asset_type][sub_asset_type] = {} for element_key, element_data in sub_asset_type_data.items(): - asset_id = element_data["GeneralInformation"]["AIM_id"] + asset_id = element_data['GeneralInformation']['AIM_id'] if asset_id != element_key: - raise ValueError( - "The rationality behidd the workdflow" - "is that oth aim-id and keys be the" - "same" + raise ValueError( # noqa: TRY003 + 'The rationality behidd the workdflow' # noqa: EM101 + 'is that oth aim-id and keys be the' + 'same' ) - self.asset_information[asset_type][sub_asset_type][ - asset_id - ] = element_data["GeneralInformation"] + self.asset_information[asset_type][sub_asset_type][asset_id] = ( + element_data['GeneralInformation'] + ) - building_state = state["Buildings"]["Building"] + building_state = state['Buildings']['Building'] for building_id, each_building in building_state.items(): - population = each_building["GeneralInformation"]["Population"] - population_ratio = each_building.get("Population_Ratio", None) + population = each_building['GeneralInformation']['Population'] + population_ratio = each_building.get('Population_Ratio', None) if population_ratio is not None: ratio = population_ratio @@ -252,14 +249,14 @@ def set_asset_data(self, state): ratio = 1 cur_building = {} - cur_building["initial_population_ratio"] = ratio - cur_building["population_ratio"] = ratio - cur_building["initial_population"] = population - cur_building["population"] = population + cur_building['initial_population_ratio'] = ratio + cur_building['population_ratio'] = ratio + cur_building['initial_population'] = population + cur_building['population'] = population self.buildings[building_id] = cur_building - def update_state_with_damages(self, damage, damage_time, state): + def update_state_with_damages(self, damage, damage_time, state): # noqa: ARG002 """ Update the state dic with damages. @@ -282,21 +279,19 @@ def update_state_with_damages(self, damage, damage_time, state): None. """ - damage = damage["WaterDistributionNetwork"] - pipe_damage = damage.get("Pipe", []) + damage = damage['WaterDistributionNetwork'] + pipe_damage = damage.get('Pipe', []) for asset_id, damage_location in pipe_damage.items(): - damage_location_info = damage_location["Damage"] + damage_location_info = damage_location['Damage'] aggregate_keys = [ - key for key in damage_location_info if "aggregate-" in key + key for key in damage_location_info if 'aggregate-' in key ] aggregate_keys.sort() - aggregate_results = [ - damage_location_info[key] for key in aggregate_keys - ] + aggregate_results = [damage_location_info[key] for key in aggregate_keys] segment_sizes = len(aggregate_results) segment_step = 1 / segment_sizes @@ -307,13 +302,11 @@ def update_state_with_damages(self, damage, damage_time, state): for damage_val in aggregate_results: if damage_val > 0: if damage_val == 1: - damage_type = "leak" - elif damage_val == 2: - damage_type = "break" + damage_type = 'leak' + elif damage_val == 2: # noqa: PLR2004 + damage_type = 'break' else: - raise ValueError( - "The damage type must be either 1 or 2" - ) + raise ValueError('The damage type must be either 1 or 2') # noqa: EM101, TRY003 else: continue @@ -321,22 +314,18 @@ def update_state_with_damages(self, damage, damage_time, state): cur_pipe_damage_location_list.append(cur_loc) cur_pipe_damage_location_type.append(damage_type) - wdn_state = state["WaterDistributionNetwork"] - pipe_state = wdn_state.get("Pipe") + wdn_state = state['WaterDistributionNetwork'] + pipe_state = wdn_state.get('Pipe') - if "Damage" in pipe_state[asset_id]: - raise ValueError( - f"Damage is already exist for Pipe " f"{asset_id}" - ) + if 'Damage' in pipe_state[asset_id]: + raise ValueError(f'Damage is already exist for Pipe ' f'{asset_id}') # noqa: EM102, TRY003 - pipe_state[asset_id]["Damage"] = dict() - pipe_state[asset_id]["Damage"][ - "Location" - ] = cur_pipe_damage_location_list + pipe_state[asset_id]['Damage'] = dict() # noqa: C408 + pipe_state[asset_id]['Damage']['Location'] = ( + cur_pipe_damage_location_list + ) - pipe_state[asset_id]["Damage"][ - "Type" - ] = cur_pipe_damage_location_type + pipe_state[asset_id]['Damage']['Type'] = cur_pipe_damage_location_type def set_rewet_damage_from_state(self, state, damage_time): """ @@ -359,25 +348,23 @@ def set_rewet_damage_from_state(self, state, damage_time): None. """ - state = state["WaterDistributionNetwork"] - pipe_damage = state.get("Pipe", []) + state = state['WaterDistributionNetwork'] + pipe_damage = state.get('Pipe', []) damage_list = [] for asset_id, pipe_info in pipe_damage.items(): - damage_location_info = pipe_info["Damage"] - damage_location_list = damage_location_info["Location"] - damage_type_list = damage_location_info["Type"] + damage_location_info = pipe_info['Damage'] + damage_location_list = damage_location_info['Location'] + damage_type_list = damage_location_info['Type'] if len(damage_location_list) != len(damage_type_list): - raise ValueError( - "The size of types and locationis not the" " same." - ) + raise ValueError('The size of types and locationis not the same.') # noqa: EM101, TRY003 segment_sizes = len(damage_location_list) - pipe_id = self.asset_information["WaterDistributionNetwork"][ - "Pipe" - ][asset_id]["InpID"] + pipe_id = self.asset_information['WaterDistributionNetwork']['Pipe'][ + asset_id + ]['InpID'] for c in range(segment_sizes): cur_loc = damage_location_list[c] @@ -385,10 +372,10 @@ def set_rewet_damage_from_state(self, state, damage_time): damage_list.append( { - "pipe_id": pipe_id, - "damage_loc": cur_loc, - "type": damage_type, - "Material": "CI", + 'pipe_id': pipe_id, + 'damage_loc': cur_loc, + 'type': damage_type, + 'Material': 'CI', } ) @@ -396,21 +383,19 @@ def set_rewet_damage_from_state(self, state, damage_time): self.pipe_damage = pd.Series( data=damage_list, index=[damage_time for val in damage_list], - dtype="O", + dtype='O', ) - self.node_damage = pd.Series(dtype="O") + self.node_damage = pd.Series(dtype='O') - self.pump_damage = pd.Series(dtype="O") + self.pump_damage = pd.Series(dtype='O') - self.tank_damage = pd.Series(dtype="O") + self.tank_damage = pd.Series(dtype='O') if damage_time in self.damage_state: - raise ValueError( - f"Time {damage_time} still exists in" f" damage state." - ) + raise ValueError(f'Time {damage_time} still exists in damage state.') # noqa: EM102, TRY003 - def couple_buildings_to_demand_nodes(self, state): + def couple_buildings_to_demand_nodes(self, state): # noqa: C901 """ Couple building to the demand nodes based on their coordinates. @@ -424,18 +409,18 @@ def couple_buildings_to_demand_nodes(self, state): None. """ - building_state = state["Buildings"]["Building"] + building_state = state['Buildings']['Building'] building_id_list = [] for building_id, each_building in building_state.items(): - location = each_building["GeneralInformation"]["location"] - coordinate = (location["latitude"], location["longitude"]) + location = each_building['GeneralInformation']['location'] + coordinate = (location['latitude'], location['longitude']) building_id_list.append(building_id) self.building_coordinates.append(coordinate) demand_node_coordinate_list = [ - val["coordinates"] for key, val in self.nodes.items() + val['coordinates'] for key, val in self.nodes.items() ] demand_node_name_list = [key for key, val in self.nodes.items()] @@ -466,17 +451,17 @@ def couple_buildings_to_demand_nodes(self, state): for node_name in self.demand_node_to_building: building_name_list = self.demand_node_to_building[node_name] population_list = [ - self.buildings[bldg_id]["initial_population"] + self.buildings[bldg_id]['initial_population'] for bldg_id in building_name_list ] total_initial_population = sum(population_list) cur_node = self.nodes[node_name] - initial_node_demand = cur_node["initial_demand"] + initial_node_demand = cur_node['initial_demand'] if initial_node_demand == 0 and total_initial_population > 0: - Warning( + Warning( # noqa: PLW0133 f"Initial demand for node {node_name} is 0." f" Thus, the demand ratio in buildidng(s)" f" {repr(building_name_list).strip('[').strip(']')}" @@ -485,7 +470,7 @@ def couple_buildings_to_demand_nodes(self, state): ) if total_initial_population == 0: - Warning( + Warning( # noqa: PLW0133 f"The population assigned to {node_name} is 0." f" Thus, the demand ratio in buildidng(s)" f" {repr(building_name_list).strip('[').strip(']')}" @@ -497,14 +482,12 @@ def couple_buildings_to_demand_nodes(self, state): # For more clarity, we name the variable "initial_population". # It does not mean that there will be ffdifferent population in # the course of recovery - self.nodes[node_name][ - "initial_population" - ] = total_initial_population + self.nodes[node_name]['initial_population'] = total_initial_population - self.nodes[node_name]["initial_node_demand"] = initial_node_demand + self.nodes[node_name]['initial_node_demand'] = initial_node_demand for bldg_id in building_name_list: - pop = self.buildings[bldg_id]["initial_population"] + pop = self.buildings[bldg_id]['initial_population'] if total_initial_population != 0: cur_bldg_initial_demand = ( @@ -513,9 +496,7 @@ def couple_buildings_to_demand_nodes(self, state): else: cur_bldg_initial_demand = None - self.buildings[bldg_id][ - "initial_demand" - ] = cur_bldg_initial_demand + self.buildings[bldg_id]['initial_demand'] = cur_bldg_initial_demand def save_damage(self, state, current_time): """ @@ -533,16 +514,16 @@ def save_damage(self, state, current_time): None. """ - pipe_damage_file_name = "temp_pipe_damage_file.pkl" - node_damage_file_name = "node_pipe_damage_file.pkl" - tank_damage_file_name = "tank_pipe_damage_file.pkl" - pump_damage_file_name = "pump_pipe_damage_file.pkl" + pipe_damage_file_name = 'temp_pipe_damage_file.pkl' + node_damage_file_name = 'node_pipe_damage_file.pkl' + tank_damage_file_name = 'tank_pipe_damage_file.pkl' + pump_damage_file_name = 'pump_pipe_damage_file.pkl' pipe_path = Path(TEMP_DIR) / pipe_damage_file_name node_path = Path(TEMP_DIR) / node_damage_file_name tank_path = Path(TEMP_DIR) / tank_damage_file_name pump_path = Path(TEMP_DIR) / pump_damage_file_name - list_path = Path(TEMP_DIR) / "list.xlsx" + list_path = Path(TEMP_DIR) / 'list.xlsx' pipe_path = str(pipe_path) node_path = str(node_path) @@ -553,15 +534,15 @@ def save_damage(self, state, current_time): self.set_rewet_damage_from_state(state, current_time) if current_time in self.damage_state: - raise ValueError( - f"The time {current_time} is already in " f" damage state." + raise ValueError( # noqa: TRY003 + f'The time {current_time} is already in ' f' damage state.' # noqa: EM102 ) self.damage_state[current_time] = { - "Pipe": self.pipe_damage, - "Node": self.node_damage, - "Pump": self.pump_damage, - "Tank": self.tank_damage, + 'Pipe': self.pipe_damage, + 'Node': self.node_damage, + 'Pump': self.pump_damage, + 'Tank': self.tank_damage, } self.pipe_damage.to_pickle(pipe_path) @@ -570,19 +551,19 @@ def save_damage(self, state, current_time): self.tank_damage.to_pickle(tank_path) scn_postfix_list = random.choices( - ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "L", "M"], k=0 + ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'L', 'M'], k=0 ) - scn_postfix = "" + scn_postfix = '' for p in scn_postfix_list: scn_postfix += p - self.damage_file_list["Scenario Name"] = "SCN_" + scn_postfix - self.damage_file_list["Pipe Damage"] = pipe_damage_file_name - self.damage_file_list["Nodal Damage"] = node_damage_file_name - self.damage_file_list["Pump Damage"] = tank_damage_file_name - self.damage_file_list["Tank Damage"] = pump_damage_file_name - self.damage_file_list["Probability"] = 1 + self.damage_file_list['Scenario Name'] = 'SCN_' + scn_postfix + self.damage_file_list['Pipe Damage'] = pipe_damage_file_name + self.damage_file_list['Nodal Damage'] = node_damage_file_name + self.damage_file_list['Pump Damage'] = tank_damage_file_name + self.damage_file_list['Tank Damage'] = pump_damage_file_name + self.damage_file_list['Probability'] = 1 damage_file_list = pd.DataFrame.from_dict([self.damage_file_list]) damage_file_list.to_excel(list_path) @@ -612,25 +593,23 @@ def run_performance(self, current_time, next_time): """ if current_time > next_time: - raise ValueError( - "Current tiime cannot be bigger than the next" "time" - ) + raise ValueError('Current tiime cannot be bigger than the next' 'time') # noqa: EM101, TRY003 if abs(next_time - current_time) % self.hydraulic_time_step != 0: - raise ValueError( - "next_time - current_time must be a factor of " - "the hydraulci time step." + raise ValueError( # noqa: TRY003 + 'next_time - current_time must be a factor of ' # noqa: EM101 + 'the hydraulci time step.' ) status = self.rewet.initiate(current_time=current_time, debug=True) if status != 0: - raise ValueError(f"There is an error: {status}") + raise ValueError(f'There is an error: {status}') # noqa: EM102, TRY003 self.rewet.apply_damage(current_time, 0.001) time_step = next_time - current_time status = self.rewet.run_hydraulic_simulation(time_step) if status != 0: - raise ValueError(f"There is an error: {status}") + raise ValueError(f'There is an error: {status}') # noqa: EM102, TRY003 return dict @@ -660,31 +639,29 @@ def make_rewet_inputs(self, current_time, next_time): list_file_path = Path(self.list_path) list_file_path = list_file_path.resolve() if not list_file_path.exists(): - raise ValueError( - f"The list file does not exists: " f"{str(list_file_path)}" + raise ValueError( # noqa: TRY003 + f'The list file does not exists: ' f'{list_file_path!s}' # noqa: EM102 ) list_file_path = str(list_file_path) temp_dir = Path(TEMP_DIR).resolve() if not temp_dir.exists(): - raise ValueError( - f"The temp directory does not exists: " f"{str(temp_dir)}" - ) + raise ValueError(f'The temp directory does not exists: ' f'{temp_dir!s}') # noqa: EM102, TRY003 temp_dir = str(temp_dir) - settings["RUN_TIME"] = run_time - settings["minimum_simulation_time"] = run_time - settings["result_directory"] = RESULT_DIR - settings["temp_directory"] = temp_dir - settings["WN_INP"] = self.inp_file_path - settings["pipe_damage_file_list"] = list_file_path - settings["pipe_damage_file_directory"] = temp_dir - settings["Restoration_on"] = False - settings["Pipe_damage_input_method"] = "pickle" - - input_file_path = Path(INPUT_FILE_DIR) / "rewet_input.json" + settings['RUN_TIME'] = run_time + settings['minimum_simulation_time'] = run_time + settings['result_directory'] = RESULT_DIR + settings['temp_directory'] = temp_dir + settings['WN_INP'] = self.inp_file_path + settings['pipe_damage_file_list'] = list_file_path + settings['pipe_damage_file_directory'] = temp_dir + settings['Restoration_on'] = False + settings['Pipe_damage_input_method'] = 'pickle' + + input_file_path = Path(INPUT_FILE_DIR) / 'rewet_input.json' input_file_path = input_file_path.resolve() - with open(input_file_path, "wt") as f: + with open(input_file_path, 'w') as f: # noqa: PTH123 json.dump(settings, f, indent=4) self.input_file_path = str(input_file_path) @@ -711,14 +688,14 @@ def get_building_data_satisfaction(self, method): """ demand_sat = self.rewet.get_satisfied_demand_ratio() - if method.upper() == "MEAN": + if method.upper() == 'MEAN': demand_sat = demand_sat.mean() - elif method.upper() == "MAX": + elif method.upper() == 'MAX': demand_sat = demand_sat.max() - elif method.upper() == "MIN": + elif method.upper() == 'MIN': demand_sat = demand_sat.min() else: - raise ValueError(f"The method is not recognizable: {method}") + raise ValueError(f'The method is not recognizable: {method}') # noqa: EM102, TRY003 building_demand_satisfaction_ratio = {} @@ -729,9 +706,7 @@ def get_building_data_satisfaction(self, method): zip(building_names, [node_demand_ratio] * len(building_names)) ) - building_demand_satisfaction_ratio.update( - cur_satisified_demand_building - ) + building_demand_satisfaction_ratio.update(cur_satisified_demand_building) return building_demand_satisfaction_ratio @@ -750,40 +725,32 @@ def set_new_demand(self, state): """ for node_name in self.demand_node_to_building: - # cur_node = self.nodes[node_name] - total_initial_population = self.nodes[node_name][ - "initial_population" - ] + total_initial_population = self.nodes[node_name]['initial_population'] if not total_initial_population > 0: continue building_name_list = self.demand_node_to_building[node_name] - building = state["Buildings"]["Building"] + building = state['Buildings']['Building'] node_new_demand = 0 for bldg_id in building_name_list: + cur_bldg_initial_demand = self.buildings[bldg_id]['initial_demand'] - cur_bldg_initial_demand = self.buildings[bldg_id][ - "initial_demand" + cur_bldg_deamnd_ratio = building[bldg_id]['GeneralInformation'][ + 'Population_Ratio' ] - cur_bldg_deamnd_ratio = building[bldg_id][ - "GeneralInformation" - ]["Population_Ratio"] - - cur_bldg_new_deamnd = ( - cur_bldg_deamnd_ratio * cur_bldg_initial_demand - ) + cur_bldg_new_deamnd = cur_bldg_deamnd_ratio * cur_bldg_initial_demand - self.buildings[bldg_id]["current_demand"] = cur_bldg_new_deamnd + self.buildings[bldg_id]['current_demand'] = cur_bldg_new_deamnd node_new_demand += cur_bldg_new_deamnd - self.nodes[node_name]["current_demand"] = node_new_demand + self.nodes[node_name]['current_demand'] = node_new_demand node = self.wn.get_node(node_name) node.demand_timeseries_list[0].base_value = node_new_demand @@ -801,17 +768,17 @@ def get_rewet_hydraulic_basic_setting(): settings = rewet.Input.Settings.Settings() settings_dict = settings.process.settings - return settings_dict + return settings_dict # noqa: RET504 -if __name__ == "__main__": - with open("Results_det.json", "rt") as f: +if __name__ == '__main__': + with open('Results_det.json') as f: # noqa: PTH123 state = json.load(f) - with open("Results_0.json", "rt") as f: + with open('Results_0.json') as f: # noqa: PTH123 damage = json.load(f) - inp_file = "waterNetwork.inp" + inp_file = 'waterNetwork.inp' interface = REWETPyReCoDes() interface.system_state(state, damage, inp_file) diff --git a/modules/performREC/pyrecodes/damage_convertor.py b/modules/performREC/pyrecodes/damage_convertor.py index a3aa1aa57..ea4f2f013 100644 --- a/modules/performREC/pyrecodes/damage_convertor.py +++ b/modules/performREC/pyrecodes/damage_convertor.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024 The Regents of the University of California +# Copyright (c) 2024 The Regents of the University of California # noqa: INP001 # Copyright (c) 2024 Leland Stanford Junior University # # This file is part of whale. @@ -39,15 +39,14 @@ import os from pathlib import Path + import pandas as pd import preprocessorIO CBIG_int = int(1e9) -def createPipeDamageInputForREWET( - pipe_damage_data, run_dir, event_time, sc_geojson -): +def createPipeDamageInputForREWET(pipe_damage_data, run_dir, event_time, sc_geojson): # noqa: N802 """ Create REWET-style piep damage file. @@ -73,42 +72,40 @@ def createPipeDamageInputForREWET( REWET-style pipe damage file. """ - pipe_id_list = [key for key in pipe_damage_data] + pipe_id_list = [key for key in pipe_damage_data] # noqa: C416 damage_list = [] damage_time = event_time sc_geojson_file = preprocessorIO.readJSONFile(sc_geojson) pipe_data = [ ss - for ss in sc_geojson_file["features"] - if ss["properties"]["type"] == "Pipe" + for ss in sc_geojson_file['features'] + if ss['properties']['type'] == 'Pipe' ] - pipe_index = [str(ss["id"]) for ss in pipe_data] - pipe_id = [ss["properties"]["InpID"] for ss in pipe_data] + pipe_index = [str(ss['id']) for ss in pipe_data] + pipe_id = [ss['properties']['InpID'] for ss in pipe_data] pipe_index_to_id = dict(zip(pipe_index, pipe_id)) for pipe_id in pipe_id_list: cur_data = pipe_damage_data[pipe_id] - cur_damage = cur_data["Damage"] + cur_damage = cur_data['Damage'] # cur_demand = cur_data["Demand"] aim_data = findAndReadAIMFile( pipe_id, - os.path.join("Results", "WaterDistributionNetwork", "Pipe"), + os.path.join('Results', 'WaterDistributionNetwork', 'Pipe'), # noqa: PTH118 run_dir, ) - material = aim_data["GeneralInformation"].get("Material", None) + material = aim_data['GeneralInformation'].get('Material', None) if material is None: # raise ValueError("Material is none") - material = "CI" + material = 'CI' aggregates_list = [ - cur_agg - for cur_agg in list(cur_damage.keys()) - if "aggregate" in cur_agg + cur_agg for cur_agg in list(cur_damage.keys()) if 'aggregate' in cur_agg ] segment_sizes = len(aggregates_list) segment_step = 1 / segment_sizes @@ -118,11 +115,11 @@ def createPipeDamageInputForREWET( damage_val = cur_damage[cur_agg] if damage_val > 0: if damage_val == 1: - damage_type = "leak" - elif damage_val == 2: - damage_type = "break" + damage_type = 'leak' + elif damage_val == 2: # noqa: PLR2004 + damage_type = 'break' else: - raise ValueError("The damage type must be eother 1 or 2") + raise ValueError('The damage type must be eother 1 or 2') # noqa: EM101, TRY003 else: continue @@ -131,24 +128,24 @@ def createPipeDamageInputForREWET( c += 1 damage_list.append( { - "pipe_id": pipe_index_to_id[pipe_id], - "damage_loc": cur_loc, - "type": damage_type, - "Material": material, + 'pipe_id': pipe_index_to_id[pipe_id], + 'damage_loc': cur_loc, + 'type': damage_type, + 'Material': material, } ) damage_list.reverse() pipe_damage_list = pd.Series( - data=damage_list, index=[damage_time for val in damage_list], dtype="O" + data=damage_list, index=[damage_time for val in damage_list], dtype='O' ) # REWET_input_data["Pipe_damage_list"] = pipe_damage_list # REWET_input_data["AIM"] = aim_data - return pipe_damage_list + return pipe_damage_list # noqa: RET504 -def createNodeDamageInputForREWET(node_damage_data, run_dir, event_time): +def createNodeDamageInputForREWET(node_damage_data, run_dir, event_time): # noqa: N802 """ Create REWET-style node damage file. @@ -167,7 +164,7 @@ def createNodeDamageInputForREWET(node_damage_data, run_dir, event_time): REWET-style node damage file. """ - node_id_list = [key for key in node_damage_data] + node_id_list = [key for key in node_damage_data] # noqa: C416 damage_list = [] damage_time = event_time @@ -175,9 +172,7 @@ def createNodeDamageInputForREWET(node_damage_data, run_dir, event_time): for node_id in node_id_list: cur_damage = node_damage_data[node_id] aggregates_list = [ - cur_agg - for cur_agg in list(cur_damage.keys()) - if "aggregate" in cur_agg + cur_agg for cur_agg in list(cur_damage.keys()) if 'aggregate' in cur_agg ] if len(aggregates_list) == 0: @@ -185,34 +180,34 @@ def createNodeDamageInputForREWET(node_damage_data, run_dir, event_time): cur_data = node_damage_data[node_id] - cur_damage = cur_data["Damage"] + cur_damage = cur_data['Damage'] # cur_demand = cur_data["Demand"] aim_data = findAndReadAIMFile( node_id, - os.path.join("Results", "WaterDistributionNetwork", "Node"), + os.path.join('Results', 'WaterDistributionNetwork', 'Node'), # noqa: PTH118 run_dir, ) - total_length = aim_data["GeneralInformation"].get("Total_length", None) - total_number_of_damages = cur_damage["aggregate"] + total_length = aim_data['GeneralInformation'].get('Total_length', None) + total_number_of_damages = cur_damage['aggregate'] damage_list.append( { - "node_name": node_id, - "number_of_damages": total_number_of_damages, - "node_Pipe_Length": total_length, + 'node_name': node_id, + 'number_of_damages': total_number_of_damages, + 'node_Pipe_Length': total_length, } ) node_damage_list = pd.Series( - data=damage_list, index=[damage_time for val in damage_list], dtype="O" + data=damage_list, index=[damage_time for val in damage_list], dtype='O' ) - return node_damage_list + return node_damage_list # noqa: RET504 -def createPumpDamageInputForREWET(pump_damage_data, REWET_input_data): +def createPumpDamageInputForREWET(pump_damage_data, REWET_input_data): # noqa: N802, N803 """ Create REWET-style pump damage file. @@ -229,16 +224,16 @@ def createPumpDamageInputForREWET(pump_damage_data, REWET_input_data): REWET-style pump damage file. """ - pump_id_list = [key for key in pump_damage_data] + pump_id_list = [key for key in pump_damage_data] # noqa: C416 damage_list = [] - damage_time = REWET_input_data["event_time"] + damage_time = REWET_input_data['event_time'] for pump_id in pump_id_list: cur_data = pump_damage_data[pump_id] - cur_damage = cur_data["Damage"] - cur_repair_time = cur_data["Repair"] + cur_damage = cur_data['Damage'] + cur_repair_time = cur_data['Repair'] if cur_damage == 0: continue # cur_damage_state = 0 means undamaged pump @@ -253,19 +248,19 @@ def createPumpDamageInputForREWET(pump_damage_data, REWET_input_data): # restore_time = getPumpRetsoreTime(cur_damage) damage_list.append( { - "pump_id": pump_id, - "time": damage_time, - "Restore_time": cur_repair_time, + 'pump_id': pump_id, + 'time': damage_time, + 'Restore_time': cur_repair_time, } ) pump_damage_list = pd.Series( index=[damage_time for val in damage_list], data=damage_list ) - return pump_damage_list + return pump_damage_list # noqa: RET504 -def createTankDamageInputForREWET(tank_damage_data, REWET_input_data): +def createTankDamageInputForREWET(tank_damage_data, REWET_input_data): # noqa: N802, N803 """ Create REWET-style Tank damage file. @@ -281,25 +276,25 @@ def createTankDamageInputForREWET(tank_damage_data, REWET_input_data): tank_damage_list : Pandas Series REWET-style tank damage file. """ - tank_id_list = [key for key in tank_damage_data] + tank_id_list = [key for key in tank_damage_data] # noqa: C416 damage_list = [] - damage_time = REWET_input_data["event_time"] + damage_time = REWET_input_data['event_time'] for tank_id in tank_id_list: cur_data = tank_damage_data[tank_id] - cur_damage = cur_data["Damage"] - cur_repair_time = cur_data["Repair"] + cur_damage = cur_data['Damage'] + cur_repair_time = cur_data['Repair'] if cur_damage == 0: continue # cur_damage_state = 0 meeans undamged tank damage_list.append( { - "tank_id": tank_id, - "time": damage_time, - "Restore_time": cur_repair_time, + 'tank_id': tank_id, + 'time': damage_time, + 'Restore_time': cur_repair_time, } ) @@ -307,10 +302,10 @@ def createTankDamageInputForREWET(tank_damage_data, REWET_input_data): index=[damage_time for val in damage_list], data=damage_list ) - return tank_damage_list + return tank_damage_list # noqa: RET504 -def findAndReadAIMFile(asset_id, asset_type, run_dir): +def findAndReadAIMFile(asset_id, asset_type, run_dir): # noqa: N802 """ Find and read the AIM file for an asset. @@ -333,15 +328,15 @@ def findAndReadAIMFile(asset_id, asset_type, run_dir): run_dir, asset_type, str(asset_id), - "templatedir", - f"{asset_id}-AIM.json", + 'templatedir', + f'{asset_id}-AIM.json', ) aim_file_data = preprocessorIO.readJSONFile(str(file_path)) - return aim_file_data + return aim_file_data # noqa: RET504 # Not UsedWE WILL GET IT FROM PELICUN -def getPumpRetsoreTime(damage_state): +def getPumpRetsoreTime(damage_state): # noqa: N802 """ Provide the restore time. @@ -365,7 +360,7 @@ def getPumpRetsoreTime(damage_state): """ if damage_state == 1: restore_time = int(3 * 24 * 3600) - elif damage_state == 2: + elif damage_state == 2: # noqa: PLR2004 restore_time = int(7 * 24 * 3600) else: restore_time = CBIG_int @@ -374,7 +369,7 @@ def getPumpRetsoreTime(damage_state): # NOT USED! WE WILL GET IT FROM PELICUN -def getTankRetsoreTime(tank_type, damage_state): +def getTankRetsoreTime(tank_type, damage_state): # noqa: ARG001, N802 """ Provide the restore time. @@ -400,7 +395,7 @@ def getTankRetsoreTime(tank_type, damage_state): """ if damage_state == 1: restore_time = int(3 * 24 * 3600) - elif damage_state == 2: + elif damage_state == 2: # noqa: PLR2004 restore_time = int(7 * 24 * 3600) else: restore_time = CBIG_int @@ -408,7 +403,7 @@ def getTankRetsoreTime(tank_type, damage_state): return restore_time -def readDamagefile(file_addr, run_dir, event_time, sc_geojson): +def readDamagefile(file_addr, run_dir, event_time, sc_geojson): # noqa: N802 """ Read PELICUN damage files. and create REWET-Style damage. @@ -436,40 +431,40 @@ def readDamagefile(file_addr, run_dir, event_time, sc_geojson): damage_data = preprocessorIO.readJSONFile(file_addr) - wn_damage_data = damage_data["WaterDistributionNetwork"] + wn_damage_data = damage_data['WaterDistributionNetwork'] - if "Pipe" in wn_damage_data: + if 'Pipe' in wn_damage_data: pipe_damage_data = createPipeDamageInputForREWET( - wn_damage_data["Pipe"], run_dir, event_time, sc_geojson + wn_damage_data['Pipe'], run_dir, event_time, sc_geojson ) else: - pipe_damage_data = pd.Series(dtype="O") + pipe_damage_data = pd.Series(dtype='O') - if "Tank" in wn_damage_data: + if 'Tank' in wn_damage_data: tank_damage_data = createTankDamageInputForREWET( - wn_damage_data["Tank"], run_dir, event_time + wn_damage_data['Tank'], run_dir, event_time ) else: - tank_damage_data = pd.Series(dtype="O") + tank_damage_data = pd.Series(dtype='O') - if "Pump" in wn_damage_data: + if 'Pump' in wn_damage_data: pump_damage_data = createPumpDamageInputForREWET( - wn_damage_data["Pump"], run_dir, event_time + wn_damage_data['Pump'], run_dir, event_time ) else: - pump_damage_data = pd.Series(dtype="O") + pump_damage_data = pd.Series(dtype='O') - if "Junction" in wn_damage_data: + if 'Junction' in wn_damage_data: node_damage_data = createNodeDamageInputForREWET( - wn_damage_data["Junction"], run_dir, event_time + wn_damage_data['Junction'], run_dir, event_time ) else: - node_damage_data = pd.Series(dtype="O") + node_damage_data = pd.Series(dtype='O') damage_data = {} - damage_data["Pipe"] = pipe_damage_data - damage_data["Tank"] = tank_damage_data - damage_data["Pump"] = pump_damage_data - damage_data["Node"] = node_damage_data + damage_data['Pipe'] = pipe_damage_data + damage_data['Tank'] = tank_damage_data + damage_data['Pump'] = pump_damage_data + damage_data['Node'] = node_damage_data return damage_data From e114e5ccf3d6a3a72d676491ddb5e4ed461b6a7c Mon Sep 17 00:00:00 2001 From: Sina Naeimi Date: Tue, 27 Aug 2024 17:03:19 -0700 Subject: [PATCH 17/26] inputs chanegd. Ruffed. --- .../pyrecodes/PyReCode-REWET-preprocessor.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/modules/performREC/pyrecodes/PyReCode-REWET-preprocessor.py b/modules/performREC/pyrecodes/PyReCode-REWET-preprocessor.py index c951ec51a..60cd9aadd 100644 --- a/modules/performREC/pyrecodes/PyReCode-REWET-preprocessor.py +++ b/modules/performREC/pyrecodes/PyReCode-REWET-preprocessor.py @@ -56,7 +56,7 @@ class REWETPyReCoDes: """Provide the wrapper for REWET API.""" - def __init__(self): + def __init__(self, inp_file): self.wn = None self._clean_wn = None self.inp_file_path = None @@ -73,8 +73,9 @@ def __init__(self): self.damage_file_list = {} self.damage_state = {} self.hydraulic_time_step = 3600 + self.inp_file_path = inp_file - def system_state(self, state, damage, inp_file, damage_time=0): + def system_state(self, state, damage, damage_time=0): """ Set the WDN system for PyReCoDes Interface. @@ -85,8 +86,6 @@ def system_state(self, state, damage, inp_file, damage_time=0): style. damage : dict The damage state in 2DTool's system_i style. - inp_file : path(str) - the path to the inp file. Damage_time : int When initil damages happen in seconds. @@ -96,7 +95,7 @@ def system_state(self, state, damage, inp_file, damage_time=0): """ # read the inp file - self.read_inp_file(inp_file) + self.read_inp_file(self.inp_file_path) self.set_asset_data(state) @@ -114,8 +113,6 @@ def system_performance(self, state, current_time, next_time): ---------- state : dict. The _det file content. - damage : dict - _i (realization) file content. current_time : int Current time in seconds. next_time : int @@ -780,6 +777,6 @@ def get_rewet_hydraulic_basic_setting(): inp_file = 'waterNetwork.inp' - interface = REWETPyReCoDes() - interface.system_state(state, damage, inp_file) - result = interface.system_performance(state, damage, 0, 24 * 3600) + interface = REWETPyReCoDes(inp_file) + interface.system_state(state, damage) + result = interface.system_performance(state, 0, 24 * 3600) From 01d6b5242f577b6ee3d0850a47bf0f90b4ec91a0 Mon Sep 17 00:00:00 2001 From: Sina Naeimi Date: Fri, 30 Aug 2024 13:35:49 -0700 Subject: [PATCH 18/26] Clean system performance and a little bit REWET Wrapper for ruff checks. --- modules/Workflow/whale/main.py | 6 +- .../systemPerformance/REWET/REWET_Wrapper.py | 156 ++++++++---------- 2 files changed, 73 insertions(+), 89 deletions(-) diff --git a/modules/Workflow/whale/main.py b/modules/Workflow/whale/main.py index 86d0f26ab..8f63ef232 100644 --- a/modules/Workflow/whale/main.py +++ b/modules/Workflow/whale/main.py @@ -1516,8 +1516,10 @@ def perform_system_performance_assessment(self, asset_type): asset_type: string Asset type to run perform system assessment of - """ # noqa: D400 - if 'SystemPerformance' in self.workflow_apps.keys(): # noqa: SIM118 + """ + + # Check if system performance is requested + if 'SystemPerformance' in self.workflow_apps: performance_app = self.workflow_apps['SystemPerformance'][asset_type] else: log_msg( diff --git a/modules/systemPerformance/REWET/REWET_Wrapper.py b/modules/systemPerformance/REWET/REWET_Wrapper.py index cbb99b361..ff6f5eccb 100644 --- a/modules/systemPerformance/REWET/REWET_Wrapper.py +++ b/modules/systemPerformance/REWET/REWET_Wrapper.py @@ -51,16 +51,10 @@ import preprocessorIO from shapely import geometry -# try: -# import REWET -# print("Imported") -# except: -# This is only for now -# print("HERE") +# TODO (SINA): please resolve this one later this_dir = Path(os.path.dirname(os.path.abspath(__file__))).resolve() # noqa: PTH100, PTH120 -# main_dir = this_dir.parent - +# TODO (SINA): Import REWET intead and check these sys.path.insert(0, str(this_dir / 'REWET')) from initial import Starter # noqa: E402 from Result_Project import Project_Result # noqa: E402 @@ -93,11 +87,7 @@ def createScnearioList(run_directory, scn_number): # noqa: N802, D103 'Probability': 1, } - scenario_list = scenario # pd.DataFrame([scenario]).set_index("Scenario Name") - - # REWET_input_data["scenario_list"] = scenario_list - - return scenario_list, prefix + return scenario, prefix def chooseARandomPreefix(damage_input_dir): # noqa: N802 @@ -142,7 +132,7 @@ def chooseARandomPreefix(damage_input_dir): # noqa: N802 Sets the settings (future project file) for REWET. REWET input data dictionary is both used as a source and destination for settinsg data. The data is stored in REWET_input_data object with key='settings'. The value - for 'settinsg' is an object of REWET's 'Settings' class. + for 'settinsg' is an object of REWET's 'Settings' class. Parameters ---------- @@ -188,50 +178,50 @@ def getDLFileName(run_dir, dl_file_path, scn_number): # noqa: N802 def setSettingsData(input_json, REWET_input_data): # noqa: ARG001, N802, N803, D103 - policy_file_name = rwhale_input_Data['SystemPerformance'][ + policy_file_name = rwhale_input_data['SystemPerformance'][ 'WaterDistributionNetwork' ]['Policy Definition'] - policy_file_path = rwhale_input_Data['SystemPerformance'][ + policy_file_path = rwhale_input_data['SystemPerformance'][ 'WaterDistributionNetwork' ]['Policy DefinitionPath'] policy_config_file = os.path.join(Path(policy_file_path), Path(policy_file_name)) # noqa: PTH118 - REWET_input_data['settings']['RUN_TIME'] = rwhale_input_Data[ + REWET_input_data['settings']['RUN_TIME'] = rwhale_input_data[ 'SystemPerformance' ]['WaterDistributionNetwork']['simulationTime'] - REWET_input_data['settings']['simulation_time_step'] = rwhale_input_Data[ + REWET_input_data['settings']['simulation_time_step'] = rwhale_input_data[ 'SystemPerformance' ]['WaterDistributionNetwork']['simulationTimeStep'] - REWET_input_data['settings']['last_sequence_termination'] = rwhale_input_Data[ + REWET_input_data['settings']['last_sequence_termination'] = rwhale_input_data[ 'SystemPerformance' ]['WaterDistributionNetwork']['last_sequence_termination'] - REWET_input_data['settings']['node_demand_temination'] = rwhale_input_Data[ + REWET_input_data['settings']['node_demand_temination'] = rwhale_input_data[ 'SystemPerformance' ]['WaterDistributionNetwork']['node_demand_temination'] - REWET_input_data['settings']['node_demand_termination_time'] = rwhale_input_Data[ + REWET_input_data['settings']['node_demand_termination_time'] = rwhale_input_data[ 'SystemPerformance' ]['WaterDistributionNetwork']['node_demand_termination_time'] REWET_input_data['settings']['node_demand_termination_ratio'] = ( - rwhale_input_Data[ + rwhale_input_data[ 'SystemPerformance' ]['WaterDistributionNetwork']['node_demand_termination_ratio'] ) - REWET_input_data['settings']['solver'] = rwhale_input_Data['SystemPerformance'][ + REWET_input_data['settings']['solver'] = rwhale_input_data['SystemPerformance'][ 'WaterDistributionNetwork' ]['Solver'] - REWET_input_data['settings']['Restoration_on'] = rwhale_input_Data[ + REWET_input_data['settings']['Restoration_on'] = rwhale_input_data[ 'SystemPerformance' ]['WaterDistributionNetwork']['Restoration_on'] - REWET_input_data['settings']['minimum_job_time'] = rwhale_input_Data[ + REWET_input_data['settings']['minimum_job_time'] = rwhale_input_data[ 'SystemPerformance' ]['WaterDistributionNetwork']['minimum_job_time'] REWET_input_data['settings']['Restortion_config_file'] = ( policy_config_file # TODO: SINA unmark it # noqa: TD002 ) - p = rwhale_input_Data['SystemPerformance']['WaterDistributionNetwork'][ + p = rwhale_input_data['SystemPerformance']['WaterDistributionNetwork'][ 'pipe_damage_model' ] REWET_input_data['settings']['pipe_damage_model'] = {} @@ -244,7 +234,7 @@ def setSettingsData(input_json, REWET_input_data): # noqa: ARG001, N802, N803, 'b': mat_data[5], } - n = rwhale_input_Data['SystemPerformance']['WaterDistributionNetwork'][ + n = rwhale_input_data['SystemPerformance']['WaterDistributionNetwork'][ 'node_damage_model' ] n = n[0] @@ -267,13 +257,13 @@ def setSettingsData(input_json, REWET_input_data): # noqa: ARG001, N802, N803, 'damage_node_model': 'equal_diameter_emitter', } - if rwhale_input_Data['SystemPerformance']['WaterDistributionNetwork'][ + if rwhale_input_data['SystemPerformance']['WaterDistributionNetwork'][ 'Pipe_Leak_Based' ]: - pipe_leak_amount = rwhale_input_Data['SystemPerformance'][ + pipe_leak_amount = rwhale_input_data['SystemPerformance'][ 'WaterDistributionNetwork' ]['pipe_leak_amount'] - pipe_leak_time = rwhale_input_Data['SystemPerformance'][ + pipe_leak_time = rwhale_input_data['SystemPerformance'][ 'WaterDistributionNetwork' ]['pipe_leak_time'] pipe_damage_discovery_mode = { @@ -282,7 +272,7 @@ def setSettingsData(input_json, REWET_input_data): # noqa: ARG001, N802, N803, 'leak_time': pipe_leak_time, } else: - pipe_time_discovery_ratio = rwhale_input_Data['SystemPerformance'][ + pipe_time_discovery_ratio = rwhale_input_data['SystemPerformance'][ 'WaterDistributionNetwork' ]['pipe_time_discovery_ratio'] pipe_damage_discovery_mode = { @@ -290,13 +280,13 @@ def setSettingsData(input_json, REWET_input_data): # noqa: ARG001, N802, N803, 'time_discovery_ratio': pipe_time_discovery_ratio, } # pd.Series([line[0] for line in pipe_time_discovery_ratio], index = [line[1] for line in pipe_time_discovery_ratio])} - if rwhale_input_Data['SystemPerformance']['WaterDistributionNetwork'][ + if rwhale_input_data['SystemPerformance']['WaterDistributionNetwork'][ 'Node_Leak_Based' ]: - node_leak_amount = rwhale_input_Data['SystemPerformance'][ + node_leak_amount = rwhale_input_data['SystemPerformance'][ 'WaterDistributionNetwork' ]['node_leak_amount'] - node_leak_time = rwhale_input_Data['SystemPerformance'][ + node_leak_time = rwhale_input_data['SystemPerformance'][ 'WaterDistributionNetwork' ]['node_leak_time'] node_damage_discovery_mode = { @@ -305,7 +295,7 @@ def setSettingsData(input_json, REWET_input_data): # noqa: ARG001, N802, N803, 'leak_time': node_leak_time, } else: - node_time_discovery_ratio = rwhale_input_Data['SystemPerformance'][ + node_time_discovery_ratio = rwhale_input_data['SystemPerformance'][ 'WaterDistributionNetwork' ]['node_time_discovery_ratio'] node_damage_discovery_mode = { @@ -313,10 +303,10 @@ def setSettingsData(input_json, REWET_input_data): # noqa: ARG001, N802, N803, 'time_discovery_ratio': node_time_discovery_ratio, } # pd.Series([line[0] for line in node_time_discovery_ratio], index = [line[1] for line in node_time_discovery_ratio])} - pump_time_discovery_ratio = rwhale_input_Data['SystemPerformance'][ + pump_time_discovery_ratio = rwhale_input_data['SystemPerformance'][ 'WaterDistributionNetwork' ]['pump_time_discovery_ratio'] - tank_time_discovery_ratio = rwhale_input_Data['SystemPerformance'][ + tank_time_discovery_ratio = rwhale_input_data['SystemPerformance'][ 'WaterDistributionNetwork' ]['tank_time_discovery_ratio'] pump_damage_discovery_model = { @@ -340,10 +330,10 @@ def setSettingsData(input_json, REWET_input_data): # noqa: ARG001, N802, N803, REWET_input_data['settings']['tank_damage_discovery_model'] = ( tank_damage_discovery_model ) - REWET_input_data['settings']['minimum_pressure'] = rwhale_input_Data[ + REWET_input_data['settings']['minimum_pressure'] = rwhale_input_data[ 'SystemPerformance' ]['WaterDistributionNetwork']['minimum_pressure'] - REWET_input_data['settings']['required_pressure'] = rwhale_input_Data[ + REWET_input_data['settings']['required_pressure'] = rwhale_input_data[ 'SystemPerformance' ]['WaterDistributionNetwork']['required_pressure'] @@ -402,29 +392,25 @@ def create_path(path): # noqa: D103 if __name__ == '__main__': # Setting arg parser - argParser = argparse.ArgumentParser('Preprocess rwhale workflow to REWET input.') # noqa: N816 + arg_parser = argparse.ArgumentParser('Preprocess rwhale workflow to REWET input.') - argParser.add_argument( + arg_parser.add_argument( '--input', '-i', default='inputRWHALE.json', help='rwhale input file json file', ) - # argParser.add_argument("--damage", "-d", - # default="water_damage_input_structure.json", - # help="water damage input json file. If provided, number of realization is ignored if prvided and number of realization is set to 1.") - - argParser.add_argument('--dir', '-d', help='WDN damage result directory') + arg_parser.add_argument('--dir', '-d', help='WDN damage result directory') - argParser.add_argument( + arg_parser.add_argument( '--number', '-n', default=None, help='If specified, indicates realization number, otherwise, all scenarios are run on all CPUS.', ) - argParser.add_argument( + arg_parser.add_argument( '--par', '-p', default=False, @@ -432,57 +418,55 @@ def create_path(path): # noqa: D103 help='if specified, uses all CPUS. 2 or more CPUs are not available, it will revert back to serial run.', ) - parser_data = argParser.parse_args() + parser_data = arg_parser.parse_args() - # learning about parallel or serial settings + # Setting parallel or serial settings - numP = 1 # noqa: N816 - procID = 0 # noqa: N816 - doParallel = False # noqa: N816 + proc_size = 1 + proc_ID = 0 + do_parallel = False mpi_spec = importlib.util.find_spec('mpi4py') found = mpi_spec is not None - if found and argParser.par: + if found and arg_parser.par: from mpi4py import MPI comm = MPI.COMM_WORLD - numP = comm.Get_size() # noqa: N816 - procID = comm.Get_rank() # noqa: N816 - if numP < 2: # noqa: PLR2004 - doParallel = False # noqa: N816 - numP = 1 # noqa: N816 - procID = 0 # noqa: N816 - print( # noqa: T201 + proc_size = comm.Get_size() + proc_ID = comm.Get_rank() + if proc_size < 2: + do_parallel = False + proc_size = 1 + proc_ID = 0 + print( 'Parallel running is not possible. Number of CPUS are are not enough.' ) else: - doParallel = True # noqa: N816 + do_parallel = True # Setting up run settings - REWET_input_data = {} REWET_input_data['settings'] = {} - # print(parser_data.input) - rwhale_input_Data = preprocessorIO.readJSONFile(parser_data.input) # noqa: N816 - setSettingsData(rwhale_input_Data, REWET_input_data) - event_time = rwhale_input_Data['SystemPerformance']['WaterDistributionNetwork'][ + rwhale_input_data = preprocessorIO.readJSONFile(parser_data.input) # noqa: N816 + setSettingsData(rwhale_input_data, REWET_input_data) + event_time = rwhale_input_data['SystemPerformance']['WaterDistributionNetwork'][ 'eventTime' ] # Set R2D environment and parameters - water_asset_data = rwhale_input_Data['Applications']['Assets'][ + water_asset_data = rwhale_input_data['Applications']['Assets'][ 'WaterDistributionNetwork' ] inp_file_addr = water_asset_data['ApplicationData']['inpFile'] if '{Current_Dir}' in inp_file_addr: inp_file_addr = inp_file_addr.replace('{Current_Dir}', '.') - sc_geojson = rwhale_input_Data['Applications']['Assets'][ + sc_geojson = rwhale_input_data['Applications']['Assets'][ 'WaterDistributionNetwork' ]['ApplicationData']['assetSourceFile'] - run_directory = rwhale_input_Data['runDir'] - number_of_realization = rwhale_input_Data['Applications']['DL'][ + run_directory = rwhale_input_data['runDir'] + number_of_realization = rwhale_input_data['Applications']['DL'][ 'WaterDistributionNetwork' ]['ApplicationData']['Realizations'] @@ -502,7 +486,7 @@ def create_path(path): # noqa: D103 damage_save_path_hir = damage_save_path create_path(damage_save_path_hir) - if parser_data.number == None: # noqa: E711 + if parser_data.number is None: scneario_list_path = damage_save_path / 'scenario_table.xlsx' else: scneario_list_path = ( @@ -517,7 +501,7 @@ def create_path(path): # noqa: D103 # Add Single Scenario or multiple scenario Damage_file_name = [] - if doParallel and procID > 0: + if do_parallel and proc_ID > 0: pass else: settings_json_file_path = preprocessorIO.saveSettingsFile( @@ -526,7 +510,7 @@ def create_path(path): # noqa: D103 scenario_table = preprocessorIO.create_scneario_table() - if parser_data.number == None: # noqa: E711 + if parser_data.number is None: Damage_file_name = list(range(number_of_realization)) else: @@ -542,7 +526,6 @@ def create_path(path): # noqa: D103 damage_data = damage_convertor.readDamagefile( dl_file_path, run_directory, event_time, sc_geojson ) - # damage_save_path = Path(run_directory) / "Results" / "WaterDistributionNetwork" / "damage_input" cur_damage_file_name_list = preprocessorIO.save_damage_data( damage_save_path, damage_data, scn_number @@ -557,10 +540,10 @@ def create_path(path): # noqa: D103 ) command = ( - 'python ' # noqa: ISC003 - + 'C:\\Users\\naeim\\Desktop\\REWET\\main.py -j ' - + str(settings_json_file_path) + 'python ' + f'C:\\Users\\naeim\\Desktop\\REWET\\main.py -j {settings_json_file_path}' ) + # try: # result = subprocess.check_output(command, shell=True, text=True) # returncode = 0 @@ -638,10 +621,9 @@ def create_path(path): # noqa: D103 for scn_name, row in p.project.scenario_list.iterrows(): # noqa: B007 realization_number = int(scn_name.strip('SCN_')) for single_requested_result in requested_result: - if ( - single_requested_result == 'DL' # noqa: PLR1714 - or single_requested_result == 'QN' - ): + + if single_requested_result in {'DL', 'QN'}: + # Running Output module's method to get DL time series status time_series_result[single_requested_result][ realization_number @@ -658,11 +640,11 @@ def create_path(path): # noqa: D103 ].index = ( time_series_result[single_requested_result][ realization_number - ].index - / 3600 + ].index / 3600 ) - # Running Output module's method to get BSC data for each junction (sum of outage) + # Run Output module's method to get BSC data for + # each junction (sum of outage) res[single_requested_result] = p.getOutageTimeGeoPandas_5( scn_name, bsc=single_requested_result, @@ -675,12 +657,12 @@ def create_path(path): # noqa: D103 res_agg[single_requested_result] = res[ single_requested_result ].to_dict() - for key in res_agg[single_requested_result].keys(): # noqa: SIM118 + for key in res_agg[single_requested_result]: res_agg[single_requested_result][key] = [ res_agg[single_requested_result][key] ] else: - for key in res_agg[single_requested_result].keys(): # noqa: SIM118 + for key in res_agg[single_requested_result]: res_agg[single_requested_result][key].append( res[single_requested_result][key] ) From beacd4671696d8e3fcb6daa87f4bf03a97b76b17 Mon Sep 17 00:00:00 2001 From: Sina Naeimi Date: Fri, 30 Aug 2024 14:33:07 -0700 Subject: [PATCH 19/26] Resolved the problem with -w result_dir in REWET Wrapper --- modules/Workflow/whale/main.py | 50 +++-------- .../systemPerformance/REWET/REWET_Wrapper.py | 82 ++++++------------- .../REWET/damage_convertor.py | 4 +- .../systemPerformance/REWET/preprocessorIO.py | 2 +- 4 files changed, 38 insertions(+), 100 deletions(-) diff --git a/modules/Workflow/whale/main.py b/modules/Workflow/whale/main.py index 8f63ef232..4175c503b 100644 --- a/modules/Workflow/whale/main.py +++ b/modules/Workflow/whale/main.py @@ -1507,9 +1507,10 @@ def augment_asset_files(self): # noqa: C901 return assetFilesList # noqa: DOC201, RUF100 def perform_system_performance_assessment(self, asset_type): - """For an asset type run the system level performance assessment application + """Run the system level performance assessment application. - Longer description + For an asset type run the system level performance assessment + application. Parameters ---------- @@ -1517,6 +1518,9 @@ def perform_system_performance_assessment(self, asset_type): Asset type to run perform system assessment of """ + # Make sure that we are in the run directory before we run REWET + # Every other path will be relative to the Run Directory (result dir) + os.chdir(self.run_dir) # Check if system performance is requested if 'SystemPerformance' in self.workflow_apps: @@ -1527,9 +1531,9 @@ def perform_system_performance_assessment(self, asset_type): prepend_timestamp=False, ) log_div() - return False # noqa: DOC201, RUF100 + return False - if performance_app.rel_path == None: # noqa: E711 + if performance_app.rel_path is None: log_msg( f'No Performance application to run for asset type: {asset_type}.', prepend_timestamp=False, @@ -1538,8 +1542,7 @@ def perform_system_performance_assessment(self, asset_type): return False log_msg( - 'Performing System Performance Application for asset type: ' - + asset_type, + f'Performing System Performance Application for asset type: {asset_type}', prepend_timestamp=False, ) log_div() @@ -1552,18 +1555,13 @@ def perform_system_performance_assessment(self, asset_type): # defaults added to a system performance app are asset_type, input_dir and running_parallel (default False) # - # app_command_list.append('--asset_type') - # app_command_list.append(asset_type) app_command_list.append('--input') app_command_list.append(self.input_file) - # app_command_list.append('--working_dir') - # app_command_list.append(self.working_dir) # Sina added this part for parallel run in REWET if self.parType == 'parSETUP': log_msg( - '\nParallel settings for System Performance for asset type:' - + asset_type, + f'\nParallel settings for System Performance for asset type:{asset_type}', prepend_timestamp=False, ) app_command_list.append('--par') @@ -1585,34 +1583,10 @@ def perform_system_performance_assessment(self, asset_type): ) log_msg( - 'System Performance Application Completed for asset type: ' + asset_type, + f'System Performance Application Completed for asset type: {asset_type}', prepend_timestamp=False, ) - # end of Sina's odifications for parallel run - - # if (self.parType == 'parSETUP'): - - # log_msg('\nWriting System Performance application for asset type:' + asset_type, prepend_timestamp=False) - # self.parCommandFile.write("\n# Writing System Performance application for asset type:" + asset_type +"\n") - - # if performance_app.runsParallel == False: - # self.parCommandFile.write(command + "\n") - # else: - # self.parCommandFile.write(self.mpiExec + " -n " + str(self.numProc) + " " + command + " --running_parallel True\n") - - # else: - - # log_msg('\n{}\n'.format(command), prepend_timestamp=False, - # prepend_blank_space=False) - - # result, returncode = run_command(command) - - # log_msg('Output: ', prepend_timestamp=False, prepend_blank_space=False) - # log_msg('\n{}\n'.format(result), prepend_timestamp=False, prepend_blank_space=False) - - # log_msg('System Performance Application Completed for asset type: ' + asset_type, prepend_timestamp=False) - log_div() return True @@ -2876,7 +2850,7 @@ def aggregate_results( # noqa: C901, PLR0912, PLR0915 bldg_dir = Path(os.path.dirname(asst_data[a_i]['file'])).resolve() # noqa: PTH120 main_dir = bldg_dir assetTypeHierarchy = [bldg_dir.name] # noqa: N806 - while main_dir.parent.name != self.run_dir.name: + while main_dir.parent.name != 'Results': main_dir = bldg_dir.parent assetTypeHierarchy = [main_dir.name] + assetTypeHierarchy # noqa: N806, RUF005 diff --git a/modules/systemPerformance/REWET/REWET_Wrapper.py b/modules/systemPerformance/REWET/REWET_Wrapper.py index ff6f5eccb..a22c80498 100644 --- a/modules/systemPerformance/REWET/REWET_Wrapper.py +++ b/modules/systemPerformance/REWET/REWET_Wrapper.py @@ -60,10 +60,8 @@ from Result_Project import Project_Result # noqa: E402 -def createScnearioList(run_directory, scn_number): # noqa: N802, D103 - damage_input_dir = os.path.join( # noqa: PTH118 - run_directory, 'Results', 'WaterDistributionNetwork', 'damage_input' - ) +def createScnearioList(run_dir, scn_number): # noqa: N802, D103 + damage_input_dir = run_dir / 'WaterDistributionNetwork' / 'damage_input' if not os.path.exists(damage_input_dir): # noqa: PTH110 os.makedirs(damage_input_dir) # noqa: PTH103 @@ -127,27 +125,6 @@ def chooseARandomPreefix(damage_input_dir): # noqa: N802 return random_prefix - # def setSettingsData(rwhale_data, REWET_input_data): - """ - Sets the settings (future project file) for REWET. REWET input data - dictionary is both used as a source and destination for settinsg data. The - data is stored in REWET_input_data object with key='settings'. The value - for 'settinsg' is an object of REWET's 'Settings' class. - - Parameters - ---------- - rwhale_data : json dict - rwhale input file. - REWET_input_data : dict - REWET input data. - - Returns - ------- - None. - - """ # noqa: RET503, W291 - - def getDLFileName(run_dir, dl_file_path, scn_number): # noqa: N802 """If dl_file_path is not given, the path is acquired from rwhale input data. @@ -167,9 +144,8 @@ def getDLFileName(run_dir, dl_file_path, scn_number): # noqa: N802 """ if dl_file_path == None: # noqa: E711 file_name = f'WaterDistributionNetwork_{scn_number}.json' - run_dir = run_dir # noqa: PLW0127 - file_dir = os.path.join(run_dir, 'Results', 'WaterDistributionNetwork') # noqa: PTH118 - file_path = os.path.join(file_dir, file_name) # noqa: PTH118 + file_dir = run_dir / 'WaterDistributionNetwork' + file_path = file_dir / file_name else: file_path = dl_file_path file_dir = Path(dl_file_path).parent @@ -443,6 +419,9 @@ def create_path(path): # noqa: D103 ) else: do_parallel = True + # get the current dicrectory, assumming the current directory + # is the run directory (Result Directory) + cur_dir = Path.cwd() # Setting up run settings REWET_input_data = {} @@ -465,24 +444,21 @@ def create_path(path): # noqa: D103 'WaterDistributionNetwork' ]['ApplicationData']['assetSourceFile'] - run_directory = rwhale_input_data['runDir'] + run_dir = cur_dir number_of_realization = rwhale_input_data['Applications']['DL'][ 'WaterDistributionNetwork' ]['ApplicationData']['Realizations'] - REWET_input_data['settings']['result_directory'] = os.path.join( # noqa: PTH118 - run_directory, 'Results', 'WaterDistributionNetwork', 'REWET_Result' - ) + rewet_res_dir = run_dir / 'WaterDistributionNetwork' / 'REWET_Result' + REWET_input_data['settings']['result_directory'] = str(rewet_res_dir) - REWET_input_data['settings']['temp_directory'] = os.path.join( # noqa: PTH118 - run_directory, 'Results', 'WaterDistributionNetwork', 'REWET_RunFiles' - ) + temp_dir = run_dir / 'WaterDistributionNetwork' / 'REWET_RunFiles' + REWET_input_data['settings']['temp_directory'] = str(temp_dir) - REWET_input_data['settings']['WN_INP'] = inp_file_addr + REWET_input_data['settings']['WN_INP'] = str(inp_file_addr) + + damage_save_path = run_dir / 'WaterDistributionNetwork' / 'damage_input' - damage_save_path = ( - Path(run_directory) / 'Results' / 'WaterDistributionNetwork' / 'damage_input' - ) damage_save_path_hir = damage_save_path create_path(damage_save_path_hir) @@ -520,11 +496,11 @@ def create_path(path): # noqa: D103 for scn_number in Damage_file_name: dl_file_path, dl_file_dir = getDLFileName( - run_directory, parser_data.dir, scn_number + run_dir, parser_data.dir, scn_number ) damage_data = damage_convertor.readDamagefile( - dl_file_path, run_directory, event_time, sc_geojson + dl_file_path, run_dir, event_time, sc_geojson ) cur_damage_file_name_list = preprocessorIO.save_damage_data( @@ -559,12 +535,7 @@ def create_path(path): # noqa: D103 create_path(REWET_input_data['settings']['result_directory']) create_path(REWET_input_data['settings']['temp_directory']) - rewet_log_path = ( - Path(run_directory) - / 'Results' - / 'WaterDistributionNetwork' - / 'rewet_log.txt' - ) + rewet_log_path = run_dir / 'WaterDistributionNetwork' / 'rewet_log.txt' system_std_out = sys.stdout with open(rewet_log_path, 'w') as log_file: # noqa: PTH123 @@ -671,10 +642,7 @@ def create_path(path): # noqa: D103 f'WaterDistributionNetwork_{realization_number}.json' ) cur_json_file_path = ( - Path(run_directory) - / 'Results' - / 'WaterDistributionNetwork' - / cur_json_file_name + run_dir / 'WaterDistributionNetwork' / cur_json_file_name ) with open(cur_json_file_path) as f: # noqa: PTH123 @@ -721,11 +689,9 @@ def create_path(path): # noqa: D103 # Append junction and reservior general information to WaterDistributionNetwork_det det_json_path = cur_json_file_path = ( - Path(run_directory) - / 'Results' - / 'WaterDistributionNetwork' - / 'WaterDistributionNetwork_det.json' + run_dir / 'WaterDistributionNetwork' / 'WaterDistributionNetwork_det.json' ) + det_json = preprocessorIO.readJSONFile(det_json_path) inp_json = preprocessorIO.readJSONFile(sc_geojson) inp_json = inp_json['features'] @@ -776,11 +742,9 @@ def create_path(path): # noqa: D103 json.dump(det_json, f, indent=2) ts_result_json_path = cur_json_file_path = ( - Path(run_directory) - / 'Results' - / 'WaterDistributionNetwork' - / 'WaterDistributionNetwork_timeseries.json' + run_dir / 'WaterDistributionNetwork' / 'WaterDistributionNetwork_timeseries.json' ) + time_series_result_struc = { 'Type': 'TimeSeries', 'Asset': 'WaterDistributionNetwork', diff --git a/modules/systemPerformance/REWET/damage_convertor.py b/modules/systemPerformance/REWET/damage_convertor.py index 9b70d66e6..d436f2390 100644 --- a/modules/systemPerformance/REWET/damage_convertor.py +++ b/modules/systemPerformance/REWET/damage_convertor.py @@ -88,7 +88,7 @@ def createPipeDamageInputForREWET(pipe_damage_data, run_dir, event_time, sc_geoj aim_data = findAndReadAIMFile( pipe_id, - os.path.join('Results', 'WaterDistributionNetwork', 'Pipe'), # noqa: PTH118 + os.path.join('WaterDistributionNetwork', 'Pipe'), # noqa: PTH118 run_dir, ) @@ -176,7 +176,7 @@ def createNodeDamageInputForREWET(node_damage_data, run_dir, event_time): # noq aim_data = findAndReadAIMFile( node_id, - os.path.join('Results', 'WaterDistributionNetwork', 'Node'), # noqa: PTH118 + os.path.join('WaterDistributionNetwork', 'Node'), # noqa: PTH118 run_dir, ) diff --git a/modules/systemPerformance/REWET/preprocessorIO.py b/modules/systemPerformance/REWET/preprocessorIO.py index 10f291d48..929d58f81 100644 --- a/modules/systemPerformance/REWET/preprocessorIO.py +++ b/modules/systemPerformance/REWET/preprocessorIO.py @@ -62,7 +62,7 @@ def readJSONFile(file_addr): # noqa: N802 """ # noqa: D401 if not os.path.exists(file_addr): # noqa: PTH110 - raise ValueError('INPUT WHALE FILE is not found.', repr(file_addr)) # noqa: EM101, TRY003 + raise ValueError('INPUT WHALE FILE is not found.', file_addr) # noqa: EM101, TRY003 with open(file_addr) as f: # noqa: PTH123 data = json.load(f) From 6185b35934d98ab2174a75456ee3b184ebc12505 Mon Sep 17 00:00:00 2001 From: Sina Naeimi Date: Fri, 30 Aug 2024 17:27:24 -0700 Subject: [PATCH 20/26] REWET WRAPPER: ruff format, check, and spell --- .../systemPerformance/REWET/REWET_Wrapper.py | 431 ++++++++++-------- .../REWET/damage_convertor.py | 192 +++----- .../systemPerformance/REWET/preprocessorIO.py | 131 ++++-- 3 files changed, 405 insertions(+), 349 deletions(-) diff --git a/modules/systemPerformance/REWET/REWET_Wrapper.py b/modules/systemPerformance/REWET/REWET_Wrapper.py index a22c80498..dc066ebdb 100644 --- a/modules/systemPerformance/REWET/REWET_Wrapper.py +++ b/modules/systemPerformance/REWET/REWET_Wrapper.py @@ -37,7 +37,7 @@ # Sina Naeimi # Jinyan Zhao -import argparse +import argparse # noqa: I001 import importlib import json import os @@ -45,6 +45,7 @@ import string import sys from pathlib import Path +import warnings import damage_convertor import pandas as pd @@ -54,42 +55,63 @@ # TODO (SINA): please resolve this one later this_dir = Path(os.path.dirname(os.path.abspath(__file__))).resolve() # noqa: PTH100, PTH120 -# TODO (SINA): Import REWET intead and check these +# TODO (SINA): Import REWET instead and check these sys.path.insert(0, str(this_dir / 'REWET')) from initial import Starter # noqa: E402 from Result_Project import Project_Result # noqa: E402 +PAR_CERITERIA = 2 -def createScnearioList(run_dir, scn_number): # noqa: N802, D103 - damage_input_dir = run_dir / 'WaterDistributionNetwork' / 'damage_input' - - if not os.path.exists(damage_input_dir): # noqa: PTH110 - os.makedirs(damage_input_dir) # noqa: PTH103 - - # REWET_input_data["damage_input_dir"] = damage_input_dir - - prefix = chooseARandomPreefix(damage_input_dir) - - scenario_name = f'{prefix}_scn_{scn_number}' - pipe_file_name = f'{prefix}_pipe_{scn_number}' - node_file_name = f'{prefix}_node_{scn_number}' - pump_file_name = f'{prefix}_pump_{scn_number}' - tank_file_name = f'{prefix}_tank_{scn_number}' - - scenario = { - 'Scenario Name': scenario_name, - 'Pipe Damage': pipe_file_name, - 'Nodal Damage': node_file_name, - 'Pump Damage': pump_file_name, - 'Tank Damage': tank_file_name, - 'Probability': 1, - } - - return scenario, prefix +# ============================================================================= +# def createScnearioList(run_dir, scn_number): +# """ +# Create scenario list. +# +# Parameters +# ---------- +# run_dir : path +# Run Directory. +# scn_number : int +# Scenario number. +# +# Returns +# ------- +# scenario : dict +# scenario. +# prefix : str +# Scenario name prefix. +# +# """ +# damage_input_dir = run_dir / 'WaterDistributionNetwork' / 'damage_input' +# damage_input_dir = damage_input_dir.resolve() +# +# if not damage_input_dir.exists(): +# damage_input_dir.mkdir(parents=True) +# +# prefix = choose_random_preefix(damage_input_dir) +# +# scenario_name = f'{prefix}_scn_{scn_number}' +# pipe_file_name = f'{prefix}_pipe_{scn_number}' +# node_file_name = f'{prefix}_node_{scn_number}' +# pump_file_name = f'{prefix}_pump_{scn_number}' +# tank_file_name = f'{prefix}_tank_{scn_number}' +# +# scenario = { +# 'Scenario Name': scenario_name, +# 'Pipe Damage': pipe_file_name, +# 'Nodal Damage': node_file_name, +# 'Pump Damage': pump_file_name, +# 'Tank Damage': tank_file_name, +# 'Probability': 1, +# } +# +# return scenario, prefix +# ============================================================================= +def choose_random_preefix(damage_input_dir): + """Choose a random prefix. -def chooseARandomPreefix(damage_input_dir): # noqa: N802 - """Choses a random prefix for sceranio and pipe, node, pump and tank damage + Choose a random prefix for sceranio and pipe, node, pump and tank damage file. The is important to find and unused prefix so if this script is being ran in parallel, then files are not being overwritten. @@ -103,7 +125,7 @@ def chooseARandomPreefix(damage_input_dir): # noqa: N802 random_prefix : str The Chosen random prefix string. - """ # noqa: D205 + """ number_of_prefix = 4 dir_list = os.listdir(damage_input_dir) @@ -125,24 +147,30 @@ def chooseARandomPreefix(damage_input_dir): # noqa: N802 return random_prefix -def getDLFileName(run_dir, dl_file_path, scn_number): # noqa: N802 - """If dl_file_path is not given, the path is acquired from rwhale input data. + +def get_dl_file_name(run_dir, dl_file_path, scn_number): + """Get damage and Loss fole name. + + If dl_file_path is not given, the path is acquired from rwhale input data. Parameters ---------- - REWET_input_data : TYPE - DESCRIPTION. - damage_file_pa : TYPE - DESCRIPTION. - scn_number : TYPE - DESCRIPTION. + rewet_input_data : dict + REWET-style settings input file. + damage_file_path : path + Path to the damage file. + scn_number : int + Damage scenario number. Returns ------- - None. + file_path : path + Path to the damage file. + file_dir : path + Path to the directory of the damage file. """ - if dl_file_path == None: # noqa: E711 + if dl_file_path is None: file_name = f'WaterDistributionNetwork_{scn_number}.json' file_dir = run_dir / 'WaterDistributionNetwork' file_path = file_dir / file_name @@ -153,56 +181,70 @@ def getDLFileName(run_dir, dl_file_path, scn_number): # noqa: N802 return file_path, file_dir -def setSettingsData(input_json, REWET_input_data): # noqa: ARG001, N802, N803, D103 - policy_file_name = rwhale_input_data['SystemPerformance'][ - 'WaterDistributionNetwork' - ]['Policy Definition'] - policy_file_path = rwhale_input_data['SystemPerformance'][ - 'WaterDistributionNetwork' - ]['Policy DefinitionPath'] +def set_settings_data(input_json, rewet_input_data): + """ + Set REWET-style input settings data. - policy_config_file = os.path.join(Path(policy_file_path), Path(policy_file_name)) # noqa: PTH118 + Parameters + ---------- + input_json : TYPE + DESCRIPTION. + rewet_input_data : TYPE + DESCRIPTION. - REWET_input_data['settings']['RUN_TIME'] = rwhale_input_data[ - 'SystemPerformance' - ]['WaterDistributionNetwork']['simulationTime'] - REWET_input_data['settings']['simulation_time_step'] = rwhale_input_data[ + Returns + ------- + None. + + """ + policy_file_name = input_json['SystemPerformance']['WaterDistributionNetwork'][ + 'Policy Definition' + ] + policy_file_path = input_json['SystemPerformance']['WaterDistributionNetwork'][ + 'Policy DefinitionPath' + ] + + policy_config_file = Path(policy_file_path) / Path(policy_file_name) + + rewet_input_data['settings']['RUN_TIME'] = input_json['SystemPerformance'][ + 'WaterDistributionNetwork' + ]['simulationTime'] + rewet_input_data['settings']['simulation_time_step'] = input_json[ 'SystemPerformance' ]['WaterDistributionNetwork']['simulationTimeStep'] - REWET_input_data['settings']['last_sequence_termination'] = rwhale_input_data[ + rewet_input_data['settings']['last_sequence_termination'] = input_json[ 'SystemPerformance' ]['WaterDistributionNetwork']['last_sequence_termination'] - REWET_input_data['settings']['node_demand_temination'] = rwhale_input_data[ + rewet_input_data['settings']['node_demand_temination'] = input_json[ 'SystemPerformance' ]['WaterDistributionNetwork']['node_demand_temination'] - REWET_input_data['settings']['node_demand_termination_time'] = rwhale_input_data[ + rewet_input_data['settings']['node_demand_termination_time'] = input_json[ 'SystemPerformance' ]['WaterDistributionNetwork']['node_demand_termination_time'] - REWET_input_data['settings']['node_demand_termination_ratio'] = ( - rwhale_input_data[ - 'SystemPerformance' - ]['WaterDistributionNetwork']['node_demand_termination_ratio'] - ) - REWET_input_data['settings']['solver'] = rwhale_input_data['SystemPerformance'][ + rewet_input_data['settings']['node_demand_termination_ratio'] = input_json[ + 'SystemPerformance' + ]['WaterDistributionNetwork']['node_demand_termination_ratio'] + rewet_input_data['settings']['solver'] = input_json['SystemPerformance'][ 'WaterDistributionNetwork' ]['Solver'] - REWET_input_data['settings']['Restoration_on'] = rwhale_input_data[ - 'SystemPerformance' - ]['WaterDistributionNetwork']['Restoration_on'] - REWET_input_data['settings']['minimum_job_time'] = rwhale_input_data[ + rewet_input_data['settings']['Restoration_on'] = input_json['SystemPerformance'][ + 'WaterDistributionNetwork' + ]['Restoration_on'] + rewet_input_data['settings']['minimum_job_time'] = input_json[ 'SystemPerformance' ]['WaterDistributionNetwork']['minimum_job_time'] - REWET_input_data['settings']['Restortion_config_file'] = ( - policy_config_file # TODO: SINA unmark it # noqa: TD002 - ) - p = rwhale_input_data['SystemPerformance']['WaterDistributionNetwork'][ + rewet_input_data['settings']['Restortion_config_file'] = str( + policy_config_file + ) # TODO (Sina): unmark it + + p = input_json['SystemPerformance']['WaterDistributionNetwork'][ 'pipe_damage_model' ] - REWET_input_data['settings']['pipe_damage_model'] = {} + rewet_input_data['settings']['pipe_damage_model'] = {} for mat_data in p: - REWET_input_data['settings']['pipe_damage_model'][mat_data[0]] = { + rewet_input_data['settings']['pipe_damage_model'][mat_data[0]] = { 'alpha': mat_data[1], 'beta': mat_data[2], 'gamma': mat_data[3], @@ -210,11 +252,11 @@ def setSettingsData(input_json, REWET_input_data): # noqa: ARG001, N802, N803, 'b': mat_data[5], } - n = rwhale_input_data['SystemPerformance']['WaterDistributionNetwork'][ + n = input_json['SystemPerformance']['WaterDistributionNetwork'][ 'node_damage_model' ] n = n[0] - REWET_input_data['settings']['node_damage_model'] = { + rewet_input_data['settings']['node_damage_model'] = { 'x': 0.9012, 'a': n[0], 'aa': n[1], @@ -233,22 +275,22 @@ def setSettingsData(input_json, REWET_input_data): # noqa: ARG001, N802, N803, 'damage_node_model': 'equal_diameter_emitter', } - if rwhale_input_data['SystemPerformance']['WaterDistributionNetwork'][ + if input_json['SystemPerformance']['WaterDistributionNetwork'][ 'Pipe_Leak_Based' ]: - pipe_leak_amount = rwhale_input_data['SystemPerformance'][ + pipe_leak_amount = input_json['SystemPerformance'][ 'WaterDistributionNetwork' ]['pipe_leak_amount'] - pipe_leak_time = rwhale_input_data['SystemPerformance'][ - 'WaterDistributionNetwork' - ]['pipe_leak_time'] + pipe_leak_time = input_json['SystemPerformance']['WaterDistributionNetwork'][ + 'pipe_leak_time' + ] pipe_damage_discovery_mode = { 'method': 'leak_based', 'leak_amount': pipe_leak_amount, 'leak_time': pipe_leak_time, } else: - pipe_time_discovery_ratio = rwhale_input_data['SystemPerformance'][ + pipe_time_discovery_ratio = input_json['SystemPerformance'][ 'WaterDistributionNetwork' ]['pipe_time_discovery_ratio'] pipe_damage_discovery_mode = { @@ -256,22 +298,22 @@ def setSettingsData(input_json, REWET_input_data): # noqa: ARG001, N802, N803, 'time_discovery_ratio': pipe_time_discovery_ratio, } # pd.Series([line[0] for line in pipe_time_discovery_ratio], index = [line[1] for line in pipe_time_discovery_ratio])} - if rwhale_input_data['SystemPerformance']['WaterDistributionNetwork'][ + if input_json['SystemPerformance']['WaterDistributionNetwork'][ 'Node_Leak_Based' ]: - node_leak_amount = rwhale_input_data['SystemPerformance'][ + node_leak_amount = input_json['SystemPerformance'][ 'WaterDistributionNetwork' ]['node_leak_amount'] - node_leak_time = rwhale_input_data['SystemPerformance'][ - 'WaterDistributionNetwork' - ]['node_leak_time'] + node_leak_time = input_json['SystemPerformance']['WaterDistributionNetwork'][ + 'node_leak_time' + ] node_damage_discovery_mode = { 'method': 'leak_based', 'leak_amount': node_leak_amount, 'leak_time': node_leak_time, } else: - node_time_discovery_ratio = rwhale_input_data['SystemPerformance'][ + node_time_discovery_ratio = input_json['SystemPerformance'][ 'WaterDistributionNetwork' ]['node_time_discovery_ratio'] node_damage_discovery_mode = { @@ -279,10 +321,10 @@ def setSettingsData(input_json, REWET_input_data): # noqa: ARG001, N802, N803, 'time_discovery_ratio': node_time_discovery_ratio, } # pd.Series([line[0] for line in node_time_discovery_ratio], index = [line[1] for line in node_time_discovery_ratio])} - pump_time_discovery_ratio = rwhale_input_data['SystemPerformance'][ + pump_time_discovery_ratio = input_json['SystemPerformance'][ 'WaterDistributionNetwork' ]['pump_time_discovery_ratio'] - tank_time_discovery_ratio = rwhale_input_data['SystemPerformance'][ + tank_time_discovery_ratio = input_json['SystemPerformance'][ 'WaterDistributionNetwork' ]['tank_time_discovery_ratio'] pump_damage_discovery_model = { @@ -294,50 +336,51 @@ def setSettingsData(input_json, REWET_input_data): # noqa: ARG001, N802, N803, 'time_discovery_ratio': tank_time_discovery_ratio, } # pd.Series([line[0] for line in tank_time_discovery_ratio], index = [line[1] for line in tank_time_discovery_ratio])} - REWET_input_data['settings']['pipe_damage_discovery_model'] = ( + rewet_input_data['settings']['pipe_damage_discovery_model'] = ( pipe_damage_discovery_mode ) - REWET_input_data['settings']['node_damage_discovery_model'] = ( + rewet_input_data['settings']['node_damage_discovery_model'] = ( node_damage_discovery_mode ) - REWET_input_data['settings']['pump_damage_discovery_model'] = ( + rewet_input_data['settings']['pump_damage_discovery_model'] = ( pump_damage_discovery_model ) - REWET_input_data['settings']['tank_damage_discovery_model'] = ( + rewet_input_data['settings']['tank_damage_discovery_model'] = ( tank_damage_discovery_model ) - REWET_input_data['settings']['minimum_pressure'] = rwhale_input_data[ + rewet_input_data['settings']['minimum_pressure'] = input_json[ 'SystemPerformance' ]['WaterDistributionNetwork']['minimum_pressure'] - REWET_input_data['settings']['required_pressure'] = rwhale_input_data[ + + rewet_input_data['settings']['required_pressure'] = input_json[ 'SystemPerformance' ]['WaterDistributionNetwork']['required_pressure'] # Not Supposed to be in R2DTool GUI ############ - REWET_input_data['settings']['minimum_simulation_time'] = ( - 0 # TODO : HERE #REWET_input_data["event_time"] + REWET_input_data["settings"]["simulation_time_step"] # noqa: TD002 - ) - REWET_input_data['settings']['save_time_step'] = True - REWET_input_data['settings']['record_restoration_agent_logs'] = True - REWET_input_data['settings']['record_damage_table_logs'] = True - REWET_input_data['settings']['simulation_time_step'] = 3600 - REWET_input_data['settings']['number_of_proccessor'] = 1 - REWET_input_data['settings']['demand_ratio'] = 1 - REWET_input_data['settings']['dmg_rst_data_save'] = True - REWET_input_data['settings']['Parameter_override'] = True - REWET_input_data['settings']['mpi_resume'] = ( + # TODO (Sina): HERE + # rewet_input_data["event_time"] + rewet_input_data["settings"]["simulation_time_step"] + rewet_input_data['settings']['minimum_simulation_time'] = 0 + rewet_input_data['settings']['save_time_step'] = True + rewet_input_data['settings']['record_restoration_agent_logs'] = True + rewet_input_data['settings']['record_damage_table_logs'] = True + rewet_input_data['settings']['simulation_time_step'] = 3600 + rewet_input_data['settings']['number_of_proccessor'] = 1 + rewet_input_data['settings']['demand_ratio'] = 1 + rewet_input_data['settings']['dmg_rst_data_save'] = True + rewet_input_data['settings']['Parameter_override'] = True + rewet_input_data['settings']['mpi_resume'] = ( True # ignores the scenarios that are done ) - REWET_input_data['settings']['ignore_empty_damage'] = False - REWET_input_data['settings']['result_details'] = 'extended' - REWET_input_data['settings']['negative_node_elmination'] = True - REWET_input_data['settings']['nne_flow_limit'] = 0.5 - REWET_input_data['settings']['nne_pressure_limit'] = -5 - REWET_input_data['settings']['Virtual_node'] = True - REWET_input_data['settings']['damage_node_model'] = ( + rewet_input_data['settings']['ignore_empty_damage'] = False + rewet_input_data['settings']['result_details'] = 'extended' + rewet_input_data['settings']['negative_node_elmination'] = True + rewet_input_data['settings']['nne_flow_limit'] = 0.5 + rewet_input_data['settings']['nne_pressure_limit'] = -5 + rewet_input_data['settings']['Virtual_node'] = True + rewet_input_data['settings']['damage_node_model'] = ( 'equal_diameter_emitter' # "equal_diameter_reservoir" ) - REWET_input_data['settings']['default_pipe_damage_model'] = { + rewet_input_data['settings']['default_pipe_damage_model'] = { 'alpha': -0.0038, 'beta': 0.1096, 'gamma': 0.0196, @@ -345,17 +388,31 @@ def setSettingsData(input_json, REWET_input_data): # noqa: ARG001, N802, N803, 'b': 1, } - REWET_input_data['settings'][ + rewet_input_data['settings'][ 'limit_result_file_size' ] = -1 # in Mb. 0 means no limit - REWET_input_data['settings']['Pipe_damage_input_method'] = 'pickle' + rewet_input_data['settings']['Pipe_damage_input_method'] = 'pickle' + +def create_path(path): + """ + Create a path. -def create_path(path): # noqa: D103 + Parameters + ---------- + path : path + Path. + + Returns + ------- + None. + + """ if isinstance(path, str): path = Path(path) not_existing_hir = [] - while os.path.exists(path) == False: # noqa: PTH110, E712 + path = path.resolve() + while path.exists() is False: not_existing_hir.append(path.name) path = path.parent @@ -368,7 +425,9 @@ def create_path(path): # noqa: D103 if __name__ == '__main__': # Setting arg parser - arg_parser = argparse.ArgumentParser('Preprocess rwhale workflow to REWET input.') + arg_parser = argparse.ArgumentParser( + 'Preprocess rwhale workflow to REWET input.' + ) arg_parser.add_argument( '--input', @@ -399,7 +458,7 @@ def create_path(path): # noqa: D103 # Setting parallel or serial settings proc_size = 1 - proc_ID = 0 + proc_id = 0 do_parallel = False mpi_spec = importlib.util.find_spec('mpi4py') @@ -409,26 +468,27 @@ def create_path(path): # noqa: D103 comm = MPI.COMM_WORLD proc_size = comm.Get_size() - proc_ID = comm.Get_rank() - if proc_size < 2: + proc_id = comm.Get_rank() + if proc_size < PAR_CERITERIA: do_parallel = False proc_size = 1 - proc_ID = 0 - print( - 'Parallel running is not possible. Number of CPUS are are not enough.' + proc_id = 0 + warnings.warn( + 'Parallel running is not possible. Number ' + 'of CPUS are are not enough.', stacklevel=2 ) else: do_parallel = True - # get the current dicrectory, assumming the current directory + # get the current directory, assuming the current directory # is the run directory (Result Directory) cur_dir = Path.cwd() # Setting up run settings - REWET_input_data = {} - REWET_input_data['settings'] = {} + rewet_input_data = {} + rewet_input_data['settings'] = {} - rwhale_input_data = preprocessorIO.readJSONFile(parser_data.input) # noqa: N816 - setSettingsData(rwhale_input_data, REWET_input_data) + rwhale_input_data = preprocessorIO.read_json_file(parser_data.input) + set_settings_data(rwhale_input_data, rewet_input_data) event_time = rwhale_input_data['SystemPerformance']['WaterDistributionNetwork'][ 'eventTime' ] @@ -450,12 +510,12 @@ def create_path(path): # noqa: D103 ]['ApplicationData']['Realizations'] rewet_res_dir = run_dir / 'WaterDistributionNetwork' / 'REWET_Result' - REWET_input_data['settings']['result_directory'] = str(rewet_res_dir) + rewet_input_data['settings']['result_directory'] = str(rewet_res_dir) temp_dir = run_dir / 'WaterDistributionNetwork' / 'REWET_RunFiles' - REWET_input_data['settings']['temp_directory'] = str(temp_dir) + rewet_input_data['settings']['temp_directory'] = str(temp_dir) - REWET_input_data['settings']['WN_INP'] = str(inp_file_addr) + rewet_input_data['settings']['WN_INP'] = str(inp_file_addr) damage_save_path = run_dir / 'WaterDistributionNetwork' / 'damage_input' @@ -469,19 +529,19 @@ def create_path(path): # noqa: D103 damage_save_path / f'scenario_table_{parser_data.number}.xlsx' ) - REWET_input_data['settings']['pipe_damage_file_list'] = str(scneario_list_path) - REWET_input_data['settings']['pipe_damage_file_directory'] = str( + rewet_input_data['settings']['pipe_damage_file_list'] = str(scneario_list_path) + rewet_input_data['settings']['pipe_damage_file_directory'] = str( damage_save_path ) # Add Single Scenario or multiple scenario Damage_file_name = [] - if do_parallel and proc_ID > 0: + if do_parallel and proc_id > 0: pass else: - settings_json_file_path = preprocessorIO.saveSettingsFile( - REWET_input_data, damage_save_path, parser_data.number + settings_json_file_path = preprocessorIO.save_settings_file( + rewet_input_data, damage_save_path, parser_data.number ) scenario_table = preprocessorIO.create_scneario_table() @@ -495,11 +555,11 @@ def create_path(path): # noqa: D103 damage_save_path = scneario_list_path.parent for scn_number in Damage_file_name: - dl_file_path, dl_file_dir = getDLFileName( + dl_file_path, dl_file_dir = get_dl_file_name( run_dir, parser_data.dir, scn_number ) - damage_data = damage_convertor.readDamagefile( + damage_data = damage_convertor.read_damage_file( dl_file_path, run_dir, event_time, sc_geojson ) @@ -512,7 +572,7 @@ def create_path(path): # noqa: D103 ) preprocessorIO.save_scenario_table( - scenario_table, REWET_input_data['settings']['pipe_damage_file_list'] + scenario_table, rewet_input_data['settings']['pipe_damage_file_list'] ) command = ( @@ -532,34 +592,34 @@ def create_path(path): # noqa: D103 # if returncode == 0: # print("REWET ran Successfully") - create_path(REWET_input_data['settings']['result_directory']) - create_path(REWET_input_data['settings']['temp_directory']) + create_path(rewet_input_data['settings']['result_directory']) + create_path(rewet_input_data['settings']['temp_directory']) rewet_log_path = run_dir / 'WaterDistributionNetwork' / 'rewet_log.txt' system_std_out = sys.stdout - with open(rewet_log_path, 'w') as log_file: # noqa: PTH123 + with rewet_log_path.open('wt') as log_file: sys.stdout = log_file REWET_starter = Starter() REWET_starter.run(settings_json_file_path) p = Project_Result( - Path(REWET_input_data['settings']['result_directory']) / 'project.prj' + Path(rewet_input_data['settings']['result_directory']) / 'project.prj' ) # these are the input for result section. They are not include in the requested_result = ['DL', 'QN'] substitute_ft = {'DL': 'Delivery', 'QN': 'Quantity'} consistency_time_window = 0 # 7200 - iConsider_leak = False # True # noqa: N816 - # the following does not matter if iConsider_leak is false + iConsider_leak = False # noqa: N816 + # the following does not matter if consider_leak is false leak_ratio = {'DL': 0.75, 'QN': 0} sub_asset_list = ['Junction', 'Pipe', 'Reservoir'] - sub_asset_name_to_id = dict() # noqa: C408 - sub_asset_id_to_name = dict() # noqa: C408 + sub_asset_name_to_id = {} + sub_asset_id_to_name = {} for sub_asset in sub_asset_list: - sc_geojson_file = preprocessorIO.readJSONFile(sc_geojson) + sc_geojson_file = preprocessorIO.read_json_file(sc_geojson) sub_asset_data = [ ss for ss in sc_geojson_file['features'] @@ -589,12 +649,10 @@ def create_path(path): # noqa: D103 ) ) - for scn_name, row in p.project.scenario_list.iterrows(): # noqa: B007 + for scn_name, _row in p.project.scenario_list.iterrows(): realization_number = int(scn_name.strip('SCN_')) for single_requested_result in requested_result: - if single_requested_result in {'DL', 'QN'}: - # Running Output module's method to get DL time series status time_series_result[single_requested_result][ realization_number @@ -611,7 +669,8 @@ def create_path(path): # noqa: D103 ].index = ( time_series_result[single_requested_result][ realization_number - ].index / 3600 + ].index + / 3600 ) # Run Output module's method to get BSC data for @@ -624,14 +683,17 @@ def create_path(path): # noqa: D103 consistency_time_window=consistency_time_window, sum_time=True, ) + if res_agg.get(single_requested_result) is None: res_agg[single_requested_result] = res[ single_requested_result ].to_dict() + for key in res_agg[single_requested_result]: res_agg[single_requested_result][key] = [ res_agg[single_requested_result][key] ] + else: for key in res_agg[single_requested_result]: res_agg[single_requested_result][key].append( @@ -645,7 +707,7 @@ def create_path(path): # noqa: D103 run_dir / 'WaterDistributionNetwork' / cur_json_file_name ) - with open(cur_json_file_path) as f: # noqa: PTH123 + with cur_json_file_path.open('rt') as f: json_data = json.load(f) for single_requested_result in requested_result: @@ -657,24 +719,24 @@ def create_path(path): # noqa: D103 'Junction', {} ) - for junction_name in req_result.keys(): # noqa: SIM118 + for junction_name in req_result.index: junction_id = sub_asset_name_to_id['Junction'][junction_name] cur_junction = junction_json_data.get(junction_id, {}) - cur_junction_SP = cur_junction.get('SystemPerformance', {}) # noqa: N816 - cur_junction_SP[result_key] = float(req_result[junction_name]) + cur_junction_sp = cur_junction.get('SystemPerformance', {}) + cur_junction_sp[result_key] = float(req_result[junction_name]) - cur_junction['SystemPerformance'] = cur_junction_SP + cur_junction['SystemPerformance'] = cur_junction_sp junction_json_data[junction_id] = cur_junction json_data['WaterDistributionNetwork']['Junction'] = ( junction_json_data ) - with open(cur_json_file_path, 'w') as f: # noqa: PTH123 + with cur_json_file_path.open('wt') as f: json_data = json.dump(json_data, f, indent=2) - res_agg_mean = dict() # noqa: C408 - res_agg_std = dict() # noqa: C408 + res_agg_mean = {} + res_agg_std = {} for single_requested_result in requested_result: res_agg[single_requested_result] = pd.DataFrame( res_agg[single_requested_result] @@ -692,29 +754,29 @@ def create_path(path): # noqa: D103 run_dir / 'WaterDistributionNetwork' / 'WaterDistributionNetwork_det.json' ) - det_json = preprocessorIO.readJSONFile(det_json_path) - inp_json = preprocessorIO.readJSONFile(sc_geojson) + det_json = preprocessorIO.read_json_file(det_json_path) + inp_json = preprocessorIO.read_json_file(sc_geojson) inp_json = inp_json['features'] for WDNtype in ['Reservoir', 'Junction']: - json_to_attach = dict() # noqa: C408 + json_to_attach = {} for ft in inp_json: prop = ft['properties'] if prop['type'] == WDNtype: - id = str(ft['id']) # noqa: A001 - generalInfo = dict() # noqa: C408, N816 + asset_id = str(ft['id']) + general_info = {} json_geometry = ft['geometry'] shapely_geometry = geometry.shape(json_geometry) wkt_geometry = shapely_geometry.wkt - generalInfo.update({'geometry': wkt_geometry}) - asset_name = sub_asset_id_to_name[WDNtype][id] - generalInfo.update({'REWET_id': asset_name}) - generalInfo.update({'AIM_id': id}) + general_info.update({'geometry': wkt_geometry}) + asset_name = sub_asset_id_to_name[WDNtype][asset_id] + general_info.update({'REWET_id': asset_name}) + general_info.update({'AIM_id': asset_id}) for key, item in prop.items(): if key == 'id': continue - generalInfo.update({key: item}) - R2Dres = dict() # noqa: C408 - asset_name = sub_asset_id_to_name[WDNtype][id] + general_info.update({key: item}) + R2Dres = {} + asset_name = sub_asset_id_to_name[WDNtype][asset_id] for single_requested_result in requested_result: if asset_name not in res_agg_mean[single_requested_result].index: continue @@ -730,19 +792,23 @@ def create_path(path): # noqa: D103 ], } ) - # location = dict() - # location.update({'latitude':ft['geometry']['coordinates'][1],\ - # 'longitude':ft['geometry']['coordinates'][0]}) - # generalInfo.update({'location':location}) + json_to_attach.update( - {id: {'GeneralInformation': generalInfo, 'R2Dres': R2Dres}} + { + asset_id: { + 'general_information': general_info, + 'R2Dres': R2Dres, + } + } ) det_json['WaterDistributionNetwork'].update({WDNtype: json_to_attach}) - with open(det_json_path, 'w') as f: # noqa: PTH123 + with det_json_path.open('wt') as f: json.dump(det_json, f, indent=2) ts_result_json_path = cur_json_file_path = ( - run_dir / 'WaterDistributionNetwork' / 'WaterDistributionNetwork_timeseries.json' + run_dir + / 'WaterDistributionNetwork' + / 'WaterDistributionNetwork_timeseries.json' ) time_series_result_struc = { @@ -760,6 +826,5 @@ def create_path(path): # noqa: D103 i ] = time_series_result[single_requested_result][i].to_dict() - with open(ts_result_json_path, 'w') as f: # noqa: PTH123 + with ts_result_json_path.open('wt') as f: json.dump(time_series_result_struc, f, indent=2) - print('here') # noqa: T201 diff --git a/modules/systemPerformance/REWET/damage_convertor.py b/modules/systemPerformance/REWET/damage_convertor.py index d436f2390..3518e1721 100644 --- a/modules/systemPerformance/REWET/damage_convertor.py +++ b/modules/systemPerformance/REWET/damage_convertor.py @@ -36,23 +36,24 @@ # Contributors: # Sina Naeimi -import os -from pathlib import Path - import pandas as pd import preprocessorIO CBIG_int = int(1e9) +LEAK_VALUE = 1 +BREAK_VALUE = 2 -def createPipeDamageInputForREWET(pipe_damage_data, run_dir, event_time, sc_geojson): # noqa: N802 - """Creates REWET-style piep damage file. +def create_pipe_damage_input_for_rewet( + pipe_damage_data, run_dir, event_time, sc_geojson +): + """Create REWET-style pipe damage file. Parameters ---------- pipe_damage_data : dict Pipe damage data from PELICUN. - REWET_input_data : dict + rewet_input_data : dict REWET input data. Raises @@ -65,12 +66,12 @@ def createPipeDamageInputForREWET(pipe_damage_data, run_dir, event_time, sc_geoj pipe_damage_list : Pandas Series REWET-style pipe damage file. - """ # noqa: D401 - pipe_id_list = [key for key in pipe_damage_data] # noqa: C416 + """ + pipe_id_list = list(pipe_damage_data.keys()) damage_list = [] damage_time = event_time - sc_geojson_file = preprocessorIO.readJSONFile(sc_geojson) + sc_geojson_file = preprocessorIO.read_json_file(sc_geojson) pipe_data = [ ss for ss in sc_geojson_file['features'] @@ -84,18 +85,18 @@ def createPipeDamageInputForREWET(pipe_damage_data, run_dir, event_time, sc_geoj cur_data = pipe_damage_data[pipe_id] cur_damage = cur_data['Damage'] - cur_demand = cur_data['Demand'] # noqa: F841 - aim_data = findAndReadAIMFile( + aim_data = find_read_aim_file( pipe_id, - os.path.join('WaterDistributionNetwork', 'Pipe'), # noqa: PTH118 + 'WaterDistributionNetwork', + 'Pipe', run_dir, ) material = aim_data['GeneralInformation'].get('Material', None) - if material == None: # noqa: E711 - # raise ValueError("Material is none") + # If material is not ptovided, then the material is CI as the default + if material is None: material = 'CI' aggregates_list = [ @@ -105,20 +106,19 @@ def createPipeDamageInputForREWET(pipe_damage_data, run_dir, event_time, sc_geoj segment_step = 1 / segment_sizes c = 0 - for cur_agg in aggregates_list: # cur_damage["aggregate"]: + for cur_agg in aggregates_list: damage_val = cur_damage[cur_agg] if damage_val > 0: - if damage_val == 1: + if damage_val == LEAK_VALUE: damage_type = 'leak' - elif damage_val == 2: # noqa: PLR2004 + elif damage_val == BREAK_VALUE: damage_type = 'break' else: - raise ValueError('The damage type must be eother 1 or 2') # noqa: EM101, TRY003 + raise ValueError('The damage type must be either 1 or 2') # noqa: EM101, TRY003 else: continue cur_loc = c * segment_step + segment_step / 2 - # print(cur_loc) c += 1 damage_list.append( { @@ -133,20 +133,17 @@ def createPipeDamageInputForREWET(pipe_damage_data, run_dir, event_time, sc_geoj data=damage_list, index=[damage_time for val in damage_list], dtype='O' ) - # REWET_input_data["Pipe_damage_list"] = pipe_damage_list - # REWET_input_data["AIM"] = aim_data - return pipe_damage_list # noqa: RET504 -def createNodeDamageInputForREWET(node_damage_data, run_dir, event_time): # noqa: N802 - """Creates REWET-style node damage file. +def create_node_damage_input_for_rewet(node_damage_data, run_dir, event_time): + """Create REWET-style node damage file. Parameters ---------- node_damage_data : dict Node damage data from PELICUN. - REWET_input_data : dict + rewet_input_data : dict REWET input data. Returns @@ -154,8 +151,8 @@ def createNodeDamageInputForREWET(node_damage_data, run_dir, event_time): # noq node_damage_list : Pandas Series REWET-style node damage file. - """ # noqa: D401 - node_id_list = [key for key in node_damage_data] # noqa: C416 + """ + node_id_list = node_damage_data.keys() damage_list = [] damage_time = event_time @@ -172,11 +169,11 @@ def createNodeDamageInputForREWET(node_damage_data, run_dir, event_time): # noq cur_data = node_damage_data[node_id] cur_damage = cur_data['Damage'] - cur_demand = cur_data['Demand'] # noqa: F841 - aim_data = findAndReadAIMFile( + aim_data = find_read_aim_file( node_id, - os.path.join('WaterDistributionNetwork', 'Node'), # noqa: PTH118 + 'WaterDistributionNetwork', + 'Node', run_dir, ) @@ -198,14 +195,14 @@ def createNodeDamageInputForREWET(node_damage_data, run_dir, event_time): # noq return node_damage_list # noqa: RET504 -def createPumpDamageInputForREWET(pump_damage_data, REWET_input_data): # noqa: N802, N803 - """Creates REWET-style pump damage file. +def create_pump_damage_input_for_rewet(pump_damage_data, rewet_input_data): + """Create REWET-style pump damage file. Parameters ---------- pump_damage_data : dict Pump damage data from PELICUN. - REWET_input_data : dict + rewet_input_data : dict REWET input data. Returns @@ -213,11 +210,11 @@ def createPumpDamageInputForREWET(pump_damage_data, REWET_input_data): # noqa: pump_damage_list : Pandas Series REWET-style pump damage file. - """ # noqa: D401 - pump_id_list = [key for key in pump_damage_data] # noqa: C416 + """ + pump_id_list = list(pump_damage_data.keys()) damage_list = [] - damage_time = REWET_input_data['event_time'] + damage_time = rewet_input_data['event_time'] for pump_id in pump_id_list: cur_data = pump_damage_data[pump_id] @@ -228,11 +225,11 @@ def createPumpDamageInputForREWET(pump_damage_data, REWET_input_data): # noqa: if cur_damage == 0: continue # cur_damage_state = 0 means undamaged pump - # I'm not sure if we need any data about the pump at this point + # (SINA) I'm not sure if we need any data about the pump at this point # aim_data = findAndReadAIMFile(tank_id, os.path.join( # "Results", "WaterDistributionNetwork", "Pump"), - # REWET_input_data["run_dir"]) + # rewet_input_data["run_dir"]) # We are getting this data from PELICUN # restore_time = getPumpRetsoreTime(cur_damage) @@ -250,14 +247,14 @@ def createPumpDamageInputForREWET(pump_damage_data, REWET_input_data): # noqa: return pump_damage_list # noqa: RET504 -def createTankDamageInputForREWET(tank_damage_data, REWET_input_data): # noqa: N802, N803 - """Creates REWET-style Tank damage file. +def create_tank_damage_input_for_rewet(tank_damage_data, rewet_input_data): + """Create REWET-style Tank damage file. Parameters ---------- tank_damage_data : dict Tank damage data from PELICUN. - REWET_input_data : dict + rewet_input_data : dict REWET input data. Returns @@ -265,11 +262,11 @@ def createTankDamageInputForREWET(tank_damage_data, REWET_input_data): # noqa: tank_damage_list : Pandas Series REWET-style tank damage file. - """ # noqa: D401 - tank_id_list = [key for key in tank_damage_data] # noqa: C416 + """ + tank_id_list = tank_damage_data.keys() damage_list = [] - damage_time = REWET_input_data['event_time'] + damage_time = rewet_input_data['event_time'] for tank_id in tank_id_list: cur_data = tank_damage_data[tank_id] @@ -285,7 +282,7 @@ def createTankDamageInputForREWET(tank_damage_data, REWET_input_data): # noqa: # # aim_data = findAndReadAIMFile(tank_id, os.path.join( # "Results", "WaterDistributionNetwork", "Tank"), - # REWET_input_data["run_dir"]) + # rewet_input_data["run_dir"]) # tank_type = aim_data["GeneralInformation"].get("Type", None) # restore_time = getTankRetsoreTime(tank_type, cur_damage) # ============================================================================= @@ -305,8 +302,8 @@ def createTankDamageInputForREWET(tank_damage_data, REWET_input_data): # noqa: return tank_damage_list # noqa: RET504 -def findAndReadAIMFile(asset_id, asset_type, run_dir): # noqa: N802 - """Finds and read the AIM file for an asset. +def find_read_aim_file(asset_id, asset_type, asset_sub_type, run_dir): + """Find and read the AIM file for an asset. Parameters ---------- @@ -322,87 +319,30 @@ def findAndReadAIMFile(asset_id, asset_type, run_dir): # noqa: N802 aim_file_data : dict AIM file data as a dict. - """ # noqa: D401 - file_path = Path( - run_dir, asset_type, str(asset_id), 'templatedir', f'{asset_id}-AIM.json' + """ + file_path = ( + run_dir + / asset_type + / asset_sub_type + / str(asset_id) + / 'templatedir' + / f'{asset_id}-AIM.json' ) - aim_file_data = preprocessorIO.readJSONFile(str(file_path)) + aim_file_data = preprocessorIO.read_json_file(str(file_path)) return aim_file_data # noqa: RET504 -def getPumpRetsoreTime(damage_state): # noqa: N802 - """NOT USED! WE WILL GET IT FROM PELICUN - - Provides the restore time based on HAZUS repair time or any other - approach available in the future. If damage state is slight, the restore - time is 3 days (in seconds). If damage state is 2, the restore time is 7 - days (in seconds). If damage state is 3 or 4, the restore time is - indefinite (a big number). - - Parameters - ---------- - damage_state : Int - Specifies the damage state (1 for slightly damages, 2 for moderate, - 3 etensive, and 4 complete. - - Returns - ------- - Retstor time : int - - - """ # noqa: D400 - if damage_state == 1: - restore_time = int(3 * 24 * 3600) - elif damage_state == 2: # noqa: PLR2004 - restore_time = int(7 * 24 * 3600) - else: - restore_time = CBIG_int - - return restore_time - +def read_damage_file(file_addr, run_dir, event_time, sc_geojson): + """Read PELICUN damage files. -def getTankRetsoreTime(tank_type, damage_state): # noqa: ARG001, N802 - """NOT USED! WE WILL GET IT FROM PELICUN - - Provides the restore time based on HAZUS repair time or any other - approach available in the future. if damage state is slight, the restore - time is 3 days (in seconds). If damage state is 2, the restore time is 7 - days (in seconds). If damage state is 3 or 4, the restore time is - indefinite (a big number). - - Parameters - ---------- - tank_type : STR - Tank type based on the data schema. The parameter is not used for now. - damage_state : Int - Specifies the damage state (1 for slightly damages, 2 for moderate, - 3 etensive, and 4 complete. - - Returns - ------- - Retstor time : int - - - """ # noqa: D400 - if damage_state == 1: - restore_time = int(3 * 24 * 3600) - elif damage_state == 2: # noqa: PLR2004 - restore_time = int(7 * 24 * 3600) - else: - restore_time = CBIG_int - - return restore_time - - -def readDamagefile(file_addr, run_dir, event_time, sc_geojson): # noqa: N802 - """Reads PELICUN damage files and create REWET-Style damage for all - WaterDistributionNetwork elements + Read PELICUN damage files and create REWET-Style damage for all + WaterDistributionNetwork elements. Parameters ---------- file_addr : path PELICUN damage file in JSON format. - REWET_input_data : dict + rewet_input_data : dict REWET input data, which is updated in the function. scn_number : dict JSON FILE. @@ -412,38 +352,36 @@ def readDamagefile(file_addr, run_dir, event_time, sc_geojson): # noqa: N802 damage_data : dict Damage data in PELICUN dict format. - """ # noqa: D205, D400, D401 - # TODO: Make reading once for each scenario # noqa: TD002 - - # wn = wntrfr.network.WaterNetworkModel(REWET_input_data["inp_file"] ) + """ + # TODO(SINA): Make reading once for each scenario - damage_data = preprocessorIO.readJSONFile(file_addr) + damage_data = preprocessorIO.read_json_file(file_addr) wn_damage_data = damage_data['WaterDistributionNetwork'] if 'Pipe' in wn_damage_data: - pipe_damage_data = createPipeDamageInputForREWET( + pipe_damage_data = create_pipe_damage_input_for_rewet( wn_damage_data['Pipe'], run_dir, event_time, sc_geojson ) else: pipe_damage_data = pd.Series(dtype='O') if 'Tank' in wn_damage_data: - tank_damage_data = createTankDamageInputForREWET( + tank_damage_data = create_tank_damage_input_for_rewet( wn_damage_data['Tank'], run_dir, event_time ) else: tank_damage_data = pd.Series(dtype='O') if 'Pump' in wn_damage_data: - pump_damage_data = createPumpDamageInputForREWET( + pump_damage_data = create_pump_damage_input_for_rewet( wn_damage_data['Pump'], run_dir, event_time ) else: pump_damage_data = pd.Series(dtype='O') if 'Junction' in wn_damage_data: - node_damage_data = createNodeDamageInputForREWET( + node_damage_data = create_node_damage_input_for_rewet( wn_damage_data['Junction'], run_dir, event_time ) else: diff --git a/modules/systemPerformance/REWET/preprocessorIO.py b/modules/systemPerformance/REWET/preprocessorIO.py index 929d58f81..8ed6d464d 100644 --- a/modules/systemPerformance/REWET/preprocessorIO.py +++ b/modules/systemPerformance/REWET/preprocessorIO.py @@ -37,13 +37,13 @@ # Sina Naeimi import json -import os +from pathlib import Path import pandas as pd -def readJSONFile(file_addr): # noqa: N802 - """Reads a json file. +def read_json_file(file_addr): + """Read a JSON file. Parameters ---------- @@ -60,33 +60,34 @@ def readJSONFile(file_addr): # noqa: N802 data : dict JSON File data as a dict. - """ # noqa: D401 - if not os.path.exists(file_addr): # noqa: PTH110 + """ + file_addr = Path(file_addr).resolve() + if not file_addr.exists(): raise ValueError('INPUT WHALE FILE is not found.', file_addr) # noqa: EM101, TRY003 - with open(file_addr) as f: # noqa: PTH123 + with file_addr.open('rt') as f: data = json.load(f) return data # noqa: RET504 # ============================================================================= -# def readRWHALEFileForREWET(file_addr, REWET_input_data): +# def readRWHALEFileForREWET(file_addr, rewet_input_data): # """ -# Reads rwhile input file and returns the data as a dict and updates REWET +# Reads rWhale input file and returns the data as a dict and updates REWET # input file. # # Parameters # ---------- # file_addr : Path -# rwhale input file path. -# REWET_input_data : dict +# rWhale input file path. +# rewet_input_data : dict # REWET input data. # # Returns # ------- # rwhale_data : dict -# rwhale inoput data as a dict. +# rWhale input data as a dict. # # """ # @@ -98,15 +99,33 @@ def readJSONFile(file_addr): # noqa: N802 # number_of_realization = rwhale_data["Applications"]\ # ["DL"]["WaterDistributionNetwork"]["ApplicationData"]["Realizations"] # -# REWET_input_data["inp_file" ] = inp_file_addr -# REWET_input_data["run_dir"] = run_directory -# REWET_input_data["number_of_realizations"] = number_of_realization +# rewet_input_data["inp_file" ] = inp_file_addr +# rewet_input_data["run_dir"] = run_directory +# rewet_input_data["number_of_realizations"] = number_of_realization # # return rwhale_data # ============================================================================= -def save_damage_data(damage_save_path, damage_data, scn_number): # noqa: D103 +def save_damage_data(damage_save_path, damage_data, scn_number): + """ + Save REWET-style damage data. + + Parameters + ---------- + damage_save_path : path + path to the damage directory. + damage_data : dict + REWET-style damage data. + scn_number : int + Scenario name. + + Returns + ------- + dict + Names of damaged files saved. + + """ pipe_damage_data = damage_data['Pipe'] node_damage_data = damage_data['Node'] pump_damage_data = damage_data['Pump'] @@ -117,10 +136,10 @@ def save_damage_data(damage_save_path, damage_data, scn_number): # noqa: D103 pump_damage_file_name = f'pump_damage_{scn_number}' tank_damage_file_name = f'tank_damage_{scn_number}' - pipe_damage_file_path = os.path.join(damage_save_path, pipe_damage_file_name) # noqa: PTH118 - node_damage_file_path = os.path.join(damage_save_path, node_damage_file_name) # noqa: PTH118 - pump_damage_file_path = os.path.join(damage_save_path, pump_damage_file_name) # noqa: PTH118 - tank_damage_file_path = os.path.join(damage_save_path, tank_damage_file_name) # noqa: PTH118 + pipe_damage_file_path = damage_save_path / pipe_damage_file_name + node_damage_file_path = damage_save_path / node_damage_file_name + pump_damage_file_path = damage_save_path / pump_damage_file_name + tank_damage_file_path = damage_save_path / tank_damage_file_name pipe_damage_data.to_pickle(pipe_damage_file_path) node_damage_data.to_pickle(node_damage_file_path) @@ -137,7 +156,16 @@ def save_damage_data(damage_save_path, damage_data, scn_number): # noqa: D103 return damage_file_name_list # noqa: RET504 -def create_scneario_table(): # noqa: D103 +def create_scneario_table(): + """ + Create a REWET-style scenario table. + + Returns + ------- + Pandas DataFrame + Scenario table. + + """ scenario_table = pd.DataFrame( dtype='O', columns=[ @@ -149,16 +177,40 @@ def create_scneario_table(): # noqa: D103 'Probability', ], ) + return scenario_table # noqa: RET504 -def update_scenario_table(scenario_table, cur_damage_file_name_list, scn_number): # noqa: D103 +def update_scenario_table(scenario_table, cur_damage_file_name_list, scn_number): + """ + Update the scenario table. + + Parameters + ---------- + scenario_table : Pandas DataFrame + Scenario table. + cur_damage_file_name_list : Dict + Damage file name. + scn_number : int + Scenario number. + + Raises + ------ + ValueError + Unknown type. + + Returns + ------- + scenario_table : List + Scenario table in the records format. + + """ if isinstance(scenario_table, pd.core.frame.DataFrame): scenario_table = scenario_table.to_dict('records') elif isinstance(scenario_table, list): pass else: - raise ValueError('This is an unknown behavior.') # noqa: EM101, TRY003, TRY004 + raise TypeError('unknown scenario table value.') # noqa: EM101, TRY003 new_row = { 'Scenario Name': f'SCN_{scn_number}', @@ -175,53 +227,54 @@ def update_scenario_table(scenario_table, cur_damage_file_name_list, scn_number) def save_scenario_table(scenario_table, scenario_table_file_path): - """Saves the scenario data including scenario table and damage data according - to the table data + """Save the scenario data. + + Save the scenario data including scenario table and damage data according + to the table data. Parameters ---------- - REWET_input_data : Dict + rewet_input_data : Dict REWET input data. Returns ------- None. - """ # noqa: D205, D400, D401, DOC202, RUF100 + """ if isinstance(scenario_table, pd.core.frame.DataFrame): pass elif isinstance(scenario_table, list): scenario_table = pd.DataFrame(scenario_table) else: - raise ValueError('This is an unknown behavior.') # noqa: EM101, TRY003, TRY004 + raise TypeError('Unknown scenario table type.') # noqa: EM101, TRY003 scenario_table = scenario_table.set_index('Scenario Name') - # scenario_list_file_path = os.path.join(damage_save_path, scenario_list_file_name) - scenario_table.to_excel(scenario_table_file_path) -def saveSettingsFile(REWET_input_data, save_directory, prefix): # noqa: N802, N803 - """Saves settings data that REWET NEEDs. +def save_settings_file(rewet_input_data, save_directory, prefix): + """Save settings data that REWET needs. Parameters ---------- - REWET_input_data : Dict + rewet_input_data : Dict REWET input data. Returns ------- - None. + setting_save_path : path + Path to the settings-file location. - """ # noqa: D401 - settings = REWET_input_data['settings'] - if prefix == None: # noqa: E711 + """ + settings = rewet_input_data['settings'] + if prefix is None: settings_file_name = 'settings.json' else: settings_file_name = prefix + '_' + 'settings.json' - damage_save_path = save_directory / settings_file_name - with open(damage_save_path, 'w') as f: # noqa: PTH123 + setting_save_path = save_directory / settings_file_name + with setting_save_path.open('wt') as f: json.dump(settings, f, indent=4) - return damage_save_path + return setting_save_path From 2976f59b3f5856e96ee9893e40e4dfb68e1751a0 Mon Sep 17 00:00:00 2001 From: bsaakash <11618528+bsaakash@users.noreply.github.com> Date: Tue, 3 Sep 2024 08:28:58 -0700 Subject: [PATCH 21/26] abs - adding missing import statement --- .../experimentalWindForces.py | 9 +++++---- .../experimentalWindPressures.py | 19 ++++++++++--------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/modules/createEVENT/experimentalWindForces/experimentalWindForces.py b/modules/createEVENT/experimentalWindForces/experimentalWindForces.py index 55a93d117..18adffd64 100644 --- a/modules/createEVENT/experimentalWindForces/experimentalWindForces.py +++ b/modules/createEVENT/experimentalWindForces/experimentalWindForces.py @@ -1,5 +1,6 @@ import json # noqa: INP001, D100 import os +import sys import time try: @@ -656,14 +657,14 @@ def err_exit(msg): # noqa: D103 # parseWindMatFile("Forces_ANG000_phase1.mat", "Forces_ANG000_phase1.json") # parseWindMatFile("TargetSpectra_ANG000_phase1.mat", "TargetSpectra_ANG000_phase1.json") - inputArgs = sys.argv # noqa: N816, F405 + inputArgs = sys.argv # noqa: N816 # set filenames - aimName = sys.argv[2] # noqa: N816, F405 - evtName = sys.argv[4] # noqa: N816, F405 + aimName = sys.argv[2] # noqa: N816 + evtName = sys.argv[4] # noqa: N816 getRV = False # noqa: N816 - for myarg in sys.argv: # noqa: F405 + for myarg in sys.argv: if myarg == '--getRV': getRV = True # noqa: N816 diff --git a/modules/createEVENT/experimentalWindPressures/experimentalWindPressures.py b/modules/createEVENT/experimentalWindPressures/experimentalWindPressures.py index b6325e8b9..66dc612df 100644 --- a/modules/createEVENT/experimentalWindPressures/experimentalWindPressures.py +++ b/modules/createEVENT/experimentalWindPressures/experimentalWindPressures.py @@ -1,5 +1,6 @@ import json # noqa: INP001, D100 import os +import sys import time try: @@ -21,7 +22,7 @@ from convertWindMat import * # noqa: F403 errPath = './workflow.err' # error file name # noqa: N816 -sys.stderr = open( # noqa: SIM115, PTH123, F405 +sys.stderr = open( # noqa: SIM115, PTH123 errPath, 'w' ) # redirecting stderr (this way we can capture all sorts of python errors) @@ -30,7 +31,7 @@ def err_exit(msg): # noqa: D103 print('Failed in wind load generator: ' + msg) # display in stdout # noqa: T201 print( 'Failed in wind load generator: ' + msg, - file=sys.stderr, # noqa: F405 + file=sys.stderr, ) # display in stderr exit(-1) # exit with non-zero exit code # noqa: PLR1722 @@ -300,7 +301,7 @@ def main(aimName, evtName, getRV): # noqa: C901, N803, D103, PLR0915 try: self.pool.shutdown() # noqa: F405 except Exception: # noqa: BLE001 - sys.exit() # noqa: F405 + sys.exit() my_cdf_vects = np.zeros((1000, tap)) my_cdf_x_range = np.zeros((2, tap)) @@ -487,7 +488,7 @@ def main(aimName, evtName, getRV): # noqa: C901, N803, D103, PLR0915 try: self.pool.shutdown() # noqa: F405 except Exception: # noqa: BLE001 - sys.exit() # noqa: F405 + sys.exit() Cp_nongauss_kernel = np.zeros((tap, CP_sim.shape[2], len(seeds))) # noqa: N806 Cp_nongauss_kernel[:, :, 0] = np.array(result_objs) @@ -834,7 +835,7 @@ def perform_POD(s_target, f_target, ncomp, l_mo, pool): # noqa: N802, D103 try: self.pool.shutdown() # noqa: F405 except Exception: # noqa: BLE001 - sys.exit() # noqa: F405 + sys.exit() for ii in range(SpeN): D_all = result_objs[ii][0] # noqa: N806 @@ -952,14 +953,14 @@ def simulation_gaussian( # noqa: D103, PLR0913 if __name__ == '__main__': - inputArgs = sys.argv # noqa: N816, F405 + inputArgs = sys.argv # noqa: N816 # set filenames - aimName = sys.argv[2] # noqa: N816, F405 - evtName = sys.argv[4] # noqa: N816, F405 + aimName = sys.argv[2] # noqa: N816 + evtName = sys.argv[4] # noqa: N816 getRV = False # noqa: N816 - for myarg in sys.argv: # noqa: F405 + for myarg in sys.argv: if (myarg == '--getRV') or (myarg == 'getRV'): # noqa: PLR1714 getRV = True # noqa: N816 From e8bfc3d6348f65865d1b612beafccbc6e3273793 Mon Sep 17 00:00:00 2001 From: bsaakash <11618528+bsaakash@users.noreply.github.com> Date: Tue, 3 Sep 2024 08:32:24 -0700 Subject: [PATCH 22/26] abs - linting --- .../experimentalWindPressures/experimentalWindPressures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/createEVENT/experimentalWindPressures/experimentalWindPressures.py b/modules/createEVENT/experimentalWindPressures/experimentalWindPressures.py index 66dc612df..667ceed68 100644 --- a/modules/createEVENT/experimentalWindPressures/experimentalWindPressures.py +++ b/modules/createEVENT/experimentalWindPressures/experimentalWindPressures.py @@ -29,7 +29,7 @@ def err_exit(msg): # noqa: D103 print('Failed in wind load generator: ' + msg) # display in stdout # noqa: T201 - print( + print( # noqa: T201 'Failed in wind load generator: ' + msg, file=sys.stderr, ) # display in stderr From a0876bfff4df0074d52f369bb6f32ab415d78576 Mon Sep 17 00:00:00 2001 From: Sina Naeimi Date: Wed, 4 Sep 2024 15:07:49 -0700 Subject: [PATCH 23/26] ruff again --- modules/systemPerformance/REWET/REWET_Wrapper.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/systemPerformance/REWET/REWET_Wrapper.py b/modules/systemPerformance/REWET/REWET_Wrapper.py index dc066ebdb..177d9a722 100644 --- a/modules/systemPerformance/REWET/REWET_Wrapper.py +++ b/modules/systemPerformance/REWET/REWET_Wrapper.py @@ -475,7 +475,8 @@ def create_path(path): proc_id = 0 warnings.warn( 'Parallel running is not possible. Number ' - 'of CPUS are are not enough.', stacklevel=2 + 'of CPUS are are not enough.', + stacklevel=2, ) else: do_parallel = True From ac8317c38076d708dd75ee4b15ac224d69aa044b Mon Sep 17 00:00:00 2001 From: bsaakash <11618528+bsaakash@users.noreply.github.com> Date: Mon, 9 Sep 2024 16:26:53 -0700 Subject: [PATCH 24/26] abs - commenting unused code --- modules/performUQ/UCSD_UQ/parseData.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/modules/performUQ/UCSD_UQ/parseData.py b/modules/performUQ/UCSD_UQ/parseData.py index d37938197..729e0d07c 100644 --- a/modules/performUQ/UCSD_UQ/parseData.py +++ b/modules/performUQ/UCSD_UQ/parseData.py @@ -35,7 +35,7 @@ def parseDataFunction(dakotaJsonFile, logFile): # noqa: C901, N802, N803, D103, applications = jsonInputs['Applications'] edpInputs = jsonInputs['EDP'] # noqa: N806 uqInputs = jsonInputs['UQ'] # noqa: N806 - femInputs = jsonInputs['FEM'] # noqa: N806, F841 + # femInputs = jsonInputs['FEM'] rvInputs = jsonInputs['randomVariables'] # noqa: N806 # localAppDirInputs = jsonInputs['localAppDir'] # pythonInputs = jsonInputs['python'] @@ -50,23 +50,23 @@ def parseDataFunction(dakotaJsonFile, logFile): # noqa: C901, N802, N803, D103, # numCol = spreadsheet['numCol'] # numRow = spreadsheet['numRow'] # summary = uqResultsInputs['summary'] - workingDir = jsonInputs['workingDir'] # noqa: N806, F841 + # workingDir = jsonInputs['workingDir'] # Processing UQ inputs logFile.write('\n\t\tProcessing UQ inputs') seedValue = uqInputs['seed'] # noqa: N806 nSamples = uqInputs['numParticles'] # noqa: N806 # maxRunTime = uqInputs["maxRunTime"] - if 'maxRunTime' in uqInputs.keys(): # noqa: SIM118 - maxRunTime = uqInputs['maxRunTime'] # noqa: N806 - else: - maxRunTime = float('inf') # noqa: N806, F841 + # if 'maxRunTime' in uqInputs.keys(): + # maxRunTime = uqInputs['maxRunTime'] + # else: + # maxRunTime = float('inf') # logLikelihoodFile = uqInputs['logLikelihoodFile'] calDataFile = uqInputs['calDataFile'] # noqa: N806 - parallelizeMCMC = True # noqa: N806 - if 'parallelExecution' in uqInputs: - parallelizeMCMC = uqInputs['parallelExecution'] # noqa: N806, F841 + # parallelizeMCMC = True + # if 'parallelExecution' in uqInputs: + # parallelizeMCMC = uqInputs['parallelExecution'] # Processing EDP inputs logFile.write('\n\n\t\tProcessing EDP inputs') @@ -90,7 +90,7 @@ def parseDataFunction(dakotaJsonFile, logFile): # noqa: C901, N802, N803, D103, logFile.write('\n\n\t\tProcessing application inputs') # Processing number of models # Check if this is a multi-model analysis - runMultiModel = False # noqa: N806, F841 + # runMultiModel = False modelsDict = {} # noqa: N806 modelIndicesList = [] # noqa: N806 modelRVNamesList = [] # noqa: N806 From 6ae6f5da94b0c22f50b16f8b66d37181b11093eb Mon Sep 17 00:00:00 2001 From: bsaakash <11618528+bsaakash@users.noreply.github.com> Date: Mon, 9 Sep 2024 16:28:54 -0700 Subject: [PATCH 25/26] abs - removing unused code --- modules/performUQ/UCSD_UQ/parseData.py | 41 -------------------------- 1 file changed, 41 deletions(-) diff --git a/modules/performUQ/UCSD_UQ/parseData.py b/modules/performUQ/UCSD_UQ/parseData.py index 729e0d07c..8ca4cc337 100644 --- a/modules/performUQ/UCSD_UQ/parseData.py +++ b/modules/performUQ/UCSD_UQ/parseData.py @@ -35,39 +35,14 @@ def parseDataFunction(dakotaJsonFile, logFile): # noqa: C901, N802, N803, D103, applications = jsonInputs['Applications'] edpInputs = jsonInputs['EDP'] # noqa: N806 uqInputs = jsonInputs['UQ'] # noqa: N806 - # femInputs = jsonInputs['FEM'] rvInputs = jsonInputs['randomVariables'] # noqa: N806 - # localAppDirInputs = jsonInputs['localAppDir'] - # pythonInputs = jsonInputs['python'] - # remoteAppDirInputs = jsonInputs['remoteAppDir'] - # uqResultsInputs = jsonInputs['uqResults'] - # if uqResultsInputs: - # resultType = uqResultsInputs['resultType'] - # if resultType == 'UCSD_Results': - # spreadsheet = uqResultsInputs['spreadsheet'] - # dataValues = spreadsheet['data'] - # headings = spreadsheet['headings'] - # numCol = spreadsheet['numCol'] - # numRow = spreadsheet['numRow'] - # summary = uqResultsInputs['summary'] - # workingDir = jsonInputs['workingDir'] # Processing UQ inputs logFile.write('\n\t\tProcessing UQ inputs') seedValue = uqInputs['seed'] # noqa: N806 nSamples = uqInputs['numParticles'] # noqa: N806 - # maxRunTime = uqInputs["maxRunTime"] - # if 'maxRunTime' in uqInputs.keys(): - # maxRunTime = uqInputs['maxRunTime'] - # else: - # maxRunTime = float('inf') - # logLikelihoodFile = uqInputs['logLikelihoodFile'] calDataFile = uqInputs['calDataFile'] # noqa: N806 - # parallelizeMCMC = True - # if 'parallelExecution' in uqInputs: - # parallelizeMCMC = uqInputs['parallelExecution'] - # Processing EDP inputs logFile.write('\n\n\t\tProcessing EDP inputs') edpNamesList = [] # noqa: N806 @@ -89,8 +64,6 @@ def parseDataFunction(dakotaJsonFile, logFile): # noqa: C901, N802, N803, D103, # Processing model inputs logFile.write('\n\n\t\tProcessing application inputs') # Processing number of models - # Check if this is a multi-model analysis - # runMultiModel = False modelsDict = {} # noqa: N806 modelIndicesList = [] # noqa: N806 modelRVNamesList = [] # noqa: N806 @@ -124,18 +97,6 @@ def parseDataFunction(dakotaJsonFile, logFile): # noqa: C901, N802, N803, D103, for _, data in modelsDict.items(): # noqa: PERF102 nModels = nModels * data['nModels'] # noqa: N806 cartesianProductOfModelIndices = list(itertools.product(*modelIndicesList)) # noqa: N806 - # logFile.write("\n\t\t\tNO LONGER Getting the number of models") - # inputFileList = [] - # nModels = femInputs['numInputs'] - # nModels = 1 - # if nModels > 1: - # fileInfo = femInputs['fileInfo'] - # for m in range(nModels): - # inputFileList.append(fileInfo[m]['inputFile']) - # else: - # inputFileList.append(femInputs['inputFile']) - # logFile.write('\n\t\t\t\tThe number of models is: {}'.format(nModels)) - # writeFEMOutputs = True # Variables variablesList = [] # noqa: N806 @@ -287,8 +248,6 @@ def parseDataFunction(dakotaJsonFile, logFile): # noqa: C901, N802, N803, D103, i, rv['name'], rv['distribution'], paramString ) ) - # if runMultiModel: - # variablesList[ind][] # Adding one prior distribution per EDP for the error covariance multiplier term logFile.write( From 7ba4db3e50f8dffd580ee19d3561ece6bae7b737 Mon Sep 17 00:00:00 2001 From: bsaakash <11618528+bsaakash@users.noreply.github.com> Date: Mon, 9 Sep 2024 16:35:20 -0700 Subject: [PATCH 26/26] abs - cleaning code --- modules/performUQ/UCSD_UQ/mainscript_tmcmc.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/modules/performUQ/UCSD_UQ/mainscript_tmcmc.py b/modules/performUQ/UCSD_UQ/mainscript_tmcmc.py index 4a9859aaf..6c9c13dd6 100644 --- a/modules/performUQ/UCSD_UQ/mainscript_tmcmc.py +++ b/modules/performUQ/UCSD_UQ/mainscript_tmcmc.py @@ -157,8 +157,6 @@ def main(input_args): # noqa: D103 ) = parseDataFunction( input_json_filename_full_path, logfile, - working_directory, - os.path.dirname(mainscript_path), # noqa: PTH120 ) syncLogFile(logfile)