Skip to content

Commit

Permalink
#810 Add the possibility to zip the inputs and outputs for N simulation
Browse files Browse the repository at this point in the history 
Signed-off-by: Dimitri Baron <[email protected]>
  • Loading branch information
@barondim
barondim committed Dec 19, 2024
1 parent 901fea6 commit b37b26e
Show file tree
Hide file tree
Showing 15 changed files with 299 additions and 51 deletions.
4 changes: 4 additions & 0 deletions etc/Dictionaries/DFLError_en_GB.dic
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,10 @@
//

//------------------ Inputs -------------------------
AlreadyInitializedNetworkFileInput = network file is already initialized
AlreadyInitializedConfigFileInput = config file is already initialized
AlreadyInitializedContingenciesInput = contingencies file is already initialized
NoConfigFileFound = no config file found
ErrorConfigFileRead = error while reading configuration file: %1%
StartingPointModeDoesntExist = starting point mode %1% doesn't exist
SettingFileWithoutAssemblingFile = %1% contains a setting file but not an assembling file
Expand Down
109 changes: 71 additions & 38 deletions scripts/envDFL.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -497,13 +497,20 @@ launch() {
if [ ! -f "$2" ]; then
error_exit "IIDM network file $2 doesn't exist"
fi
if [ ! -f "$3" ]; then
error_exit "DFL configuration file $3 doesn't exist"

if [[ "$2" == *.zip ]]; then
$DYNAFLOW_LAUNCHER_INSTALL_DIR/bin/DynaFlowLauncher \
--log-level $DYNAFLOW_LAUNCHER_LOG_LEVEL \
--input-archive $2
else
if [ ! -f "$3" ]; then
error_exit "DFL configuration file $3 doesn't exist"
fi
$DYNAFLOW_LAUNCHER_INSTALL_DIR/bin/DynaFlowLauncher \
--log-level $DYNAFLOW_LAUNCHER_LOG_LEVEL \
--network $2 \
--config $3
fi
$DYNAFLOW_LAUNCHER_INSTALL_DIR/bin/DynaFlowLauncher \
--log-level $DYNAFLOW_LAUNCHER_LOG_LEVEL \
--network $2 \
--config $3
}

launch_gdb() {
Expand Down Expand Up @@ -537,23 +544,34 @@ launch_sa() {
if [ ! -f "$2" ]; then
error_exit "IIDM network file $2 doesn't exist"
fi
if [ ! -f "$3" ]; then
error_exit "DFL configuration file $3 doesn't exist"
fi
if [ ! -f "$4" ]; then
error_exit "Security Analysis contingencies file $4 doesn't exist"
fi

export_preload
if [ "${DYNAWO_USE_MPI}" == "YES" ]; then
"$MPIRUN_PATH" -np $NBPROCS $DYNAFLOW_LAUNCHER_INSTALL_DIR/bin/DynaFlowLauncher --log-level $DYNAFLOW_LAUNCHER_LOG_LEVEL \
--network $2 \
--config $3 \
--contingencies $4
if [[ "$2" == *.zip ]]; then
if [ "${DYNAWO_USE_MPI}" == "YES" ]; then
"$MPIRUN_PATH" -np $NBPROCS $DYNAFLOW_LAUNCHER_INSTALL_DIR/bin/DynaFlowLauncher --log-level $DYNAFLOW_LAUNCHER_LOG_LEVEL \
--input-archive $2
else
$DYNAFLOW_LAUNCHER_INSTALL_DIR/bin/DynaFlowLauncher --log-level $DYNAFLOW_LAUNCHER_LOG_LEVEL \
--input-archive $2
fi
else
$DYNAFLOW_LAUNCHER_INSTALL_DIR/bin/DynaFlowLauncher --log-level $DYNAFLOW_LAUNCHER_LOG_LEVEL \
--network $2 \
--config $3 \
--contingencies $4
if [ ! -f "$3" ]; then
error_exit "DFL configuration file $3 doesn't exist"
fi
if [ ! -f "$4" ]; then
error_exit "Security Analysis contingencies file $4 doesn't exist"
fi
if [ "${DYNAWO_USE_MPI}" == "YES" ]; then
"$MPIRUN_PATH" -np $NBPROCS $DYNAFLOW_LAUNCHER_INSTALL_DIR/bin/DynaFlowLauncher --log-level $DYNAFLOW_LAUNCHER_LOG_LEVEL \
--network $2 \
--config $3 \
--contingencies $4
else
$DYNAFLOW_LAUNCHER_INSTALL_DIR/bin/DynaFlowLauncher --log-level $DYNAFLOW_LAUNCHER_LOG_LEVEL \
--network $2 \
--config $3 \
--contingencies $4
fi
fi
unset LD_PRELOAD
}
Expand All @@ -562,25 +580,40 @@ launch_nsa() {
if [ ! -f "$2" ]; then
error_exit "IIDM network file $2 doesn't exist"
fi
if [ ! -f "$3" ]; then
error_exit "DFL configuration file $3 doesn't exist"
fi
if [ ! -f "$4" ]; then
error_exit "Security Analysis contingencies file $4 doesn't exist"
fi

export_preload
if [ "${DYNAWO_USE_MPI}" == "YES" ]; then
"$MPIRUN_PATH" -np $NBPROCS $DYNAFLOW_LAUNCHER_INSTALL_DIR/bin/DynaFlowLauncher --log-level $DYNAFLOW_LAUNCHER_LOG_LEVEL \
--network $2 \
--config $3 \
--contingencies $4 \
--nsa # Steady state calculations.
if [[ "$2" == *.zip ]]; then
if [ "${DYNAWO_USE_MPI}" == "YES" ]; then
"$MPIRUN_PATH" -np $NBPROCS $DYNAFLOW_LAUNCHER_INSTALL_DIR/bin/DynaFlowLauncher --log-level $DYNAFLOW_LAUNCHER_LOG_LEVEL \
--input-archive $2 \
--nsa # Steady state calculations.
else
$DYNAFLOW_LAUNCHER_INSTALL_DIR/bin/DynaFlowLauncher --log-level $DYNAFLOW_LAUNCHER_LOG_LEVEL \
--network $2 \
--config $3 \
--contingencies $4 \
--nsa # Steady state calculations.
fi
else
$DYNAFLOW_LAUNCHER_INSTALL_DIR/bin/DynaFlowLauncher --log-level $DYNAFLOW_LAUNCHER_LOG_LEVEL \
--network $2 \
--config $3 \
--contingencies $4 \
--nsa # Steady state calculations.
if [ ! -f "$3" ]; then
error_exit "DFL configuration file $3 doesn't exist"
fi
if [ ! -f "$4" ]; then
error_exit "Security Analysis contingencies file $4 doesn't exist"
fi
if [ "${DYNAWO_USE_MPI}" == "YES" ]; then
"$MPIRUN_PATH" -np $NBPROCS $DYNAFLOW_LAUNCHER_INSTALL_DIR/bin/DynaFlowLauncher --log-level $DYNAFLOW_LAUNCHER_LOG_LEVEL \
--network $2 \
--config $3 \
--contingencies $4 \
--nsa # Steady state calculations.
else
$DYNAFLOW_LAUNCHER_INSTALL_DIR/bin/DynaFlowLauncher --log-level $DYNAFLOW_LAUNCHER_LOG_LEVEL \
--network $2 \
--config $3 \
--contingencies $4 \
--nsa # Steady state calculations.
fi
fi
unset LD_PRELOAD
}
Expand Down
1 change: 1 addition & 0 deletions sources/Common/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ target_link_libraries(dfl_Common

PRIVATE
Boost::filesystem
libZIP::libZIP
)
add_library(DynaFlowLauncher::common ALIAS dfl_Common)
install(FILES ${CMAKE_SOURCE_DIR}/etc/Dictionaries/DFLLog_en_GB.dic ${CMAKE_SOURCE_DIR}/etc/Dictionaries/DFLError_en_GB.dic DESTINATION share)
Expand Down
2 changes: 1 addition & 1 deletion sources/Common/include/Options.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@

#pragma once

#include <boost/optional.hpp>
#include <boost/program_options.hpp>
#include <iostream>
#include <string>
Expand Down Expand Up @@ -44,6 +43,7 @@ class Options {
std::string contingenciesFilePath; ///< Contingencies filepath for security analysis
std::string configPath; ///< Launcher configuration filepath
std::string dynawoLogLevel; ///< chosen log level
std::string zipArchive;
};

/**
Expand Down
111 changes: 100 additions & 11 deletions sources/Common/src/Options.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,20 @@
*/

#include "Options.h"

#include "Log.h"
#include "version.h"

#include <libzip/ZipInputStream.h>

#include <DYNFileSystemUtils.h>

#include <algorithm>
#include <boost/filesystem.hpp>
#include <boost/property_tree/json_parser.hpp>
#include <boost/property_tree/ptree.hpp>
#include <sstream>


namespace dfl {
namespace common {

Expand Down Expand Up @@ -81,14 +88,16 @@ Options::basename(const std::string& filepath) {
return path.filename().replace_extension().generic_string();
}

Options::Options() : desc_{}, config_{"", "", "", "", defaultLogLevel_} {
desc_.add_options()("help,h", "Display help message")(
"log-level", po::value<ParsedLogLevel>(),
(std::string("Dynawo logger level (allowed values are ERROR, WARN, INFO, DEBUG): default is ") + defaultLogLevel_).c_str())(
"network", po::value<std::string>(&config_.networkFilePath)->required(), "Network file path to process (IIDM support only)")(
"contingencies", po::value<std::string>(&config_.contingenciesFilePath), "Contingencies file path to process (Security Analysis)")(
"config", po::value<std::string>(&config_.configPath)->required(), "launcher Configuration file to use")("version,v", "Display version")(
"nsa", "Run steady state calculation followed by security analysis. Requires contingencies file to be defined.");
Options::Options() : desc_{}, config_{"", "", "", "", defaultLogLevel_, ""} {
desc_.add_options()
("help,h", "Display help message")
("log-level", po::value<ParsedLogLevel>(),
(std::string("Dynawo logger level (allowed values are ERROR, WARN, INFO, DEBUG): default is ") + defaultLogLevel_).c_str())
("network", po::value<std::string>(&config_.networkFilePath), "Network file path to process (IIDM support only)")
("contingencies", po::value<std::string>(&config_.contingenciesFilePath), "Contingencies file path to process (Security Analysis)")
("config", po::value<std::string>(&config_.configPath), "launcher Configuration file to use")
("version,v", "Display version")("nsa", "Run steady state calculation followed by security analysis. Requires contingencies file to be defined.")
("input-archive", po::value<std::string>(&config_.zipArchive), "zip archive");
}

Options::Request
Expand All @@ -106,24 +115,104 @@ Options::parse(int argc, char* argv[]) {
return Request::VERSION;
}

if (!vm.count("network") && !vm.count("config") && !vm.count("input-archive")) {
DYN::Trace::error(dfl::common::Log::getTag()) << "No input provided" << DYN::Trace::endline;
return Request::HELP;
}

int option_error = false;
if ((vm.count("network") && !vm.count("config")) || (vm.count("config") && !vm.count("network"))) {
option_error = true;
DYN::Trace::error(dfl::common::Log::getTag()) << "--network and --config must be used together" << DYN::Trace::endline;
}

if ((vm.count("network") || vm.count("config")) && vm.count("input-archive")) {
option_error = true;
DYN::Trace::error(dfl::common::Log::getTag()) << "--network and --config options can't be used with --input-archive" << DYN::Trace::endline;
}

po::notify(vm);

if (option_error) {
return Request::ERROR;
}

if (!config_.networkFilePath.empty() && !config_.configPath.empty()) {
// nothing to do
} else if (!config_.zipArchive.empty()) {
std::vector<std::string> zipArchiveFiles;
boost::shared_ptr<zip::ZipFile> archive = zip::ZipInputStream::read(config_.zipArchive);
std::string archiveParentPath = boost::filesystem::path(config_.zipArchive).parent_path().string();
for (std::map<std::string, boost::shared_ptr<zip::ZipEntry>>::const_iterator itE = archive->getEntries().begin();
itE != archive->getEntries().end(); ++itE) {
std::string name = itE->first;
std::string data(itE->second->getData());
std::ofstream file;
std::string filepath = createAbsolutePath(name, archiveParentPath);
file.open(createAbsolutePath(name, archiveParentPath).c_str(), std::ios::binary);
file << data;
file.close();
zipArchiveFiles.push_back(filepath);
}
for (const std::string& zipArchiveFile : zipArchiveFiles) {
if (extensionEquals(zipArchiveFile, ".iidm")) {
if (!config_.networkFilePath.empty()) {
throw Error(AlreadyInitializedNetworkFileInput);
}
config_.networkFilePath = zipArchiveFile;
} else if (extensionEquals(zipArchiveFile, ".json")) {
boost::property_tree::ptree tree;
boost::property_tree::read_json(zipArchiveFile, tree);

boost::property_tree::ptree::const_assoc_iterator configTreeIt = tree.find("dfl-config");
if (configTreeIt != tree.not_found()) {
if (!config_.configPath.empty()) {
throw Error(AlreadyInitializedConfigFileInput);
}
config_.configPath = zipArchiveFile;
}

boost::property_tree::ptree::const_assoc_iterator contingenciesTreeIt = tree.find("contingencies");
if (contingenciesTreeIt != tree.not_found()) {
if (!config_.contingenciesFilePath.empty()) {
throw Error(AlreadyInitializedContingenciesInput);
}
config_.contingenciesFilePath = zipArchiveFile;
}
} else {
throw Error(ErrorConfigFileRead, zipArchiveFile);
}
}
} else {
throw std::logic_error("Wrong boolean value.");
}

if (config_.configPath.empty()) {
throw Error(NoConfigFileFound);
}

// if (config_.contingenciesFilePath.empty()) {
// throw std::runtime_error("ERREUR");
// }

// These are not binded automatically
if (vm.count("log-level") > 0) {
config_.dynawoLogLevel = vm["log-level"].as<ParsedLogLevel>().logLevelDefinition;
}

if (vm.count("nsa") > 0) {
if (vm.count("contingencies") > 0) {
if (!config_.contingenciesFilePath.empty()) {
return Request::RUN_SIMULATION_NSA;
} else {
return Request::ERROR;
}
} else if (vm.count("contingencies") > 0) {
} else if (!config_.contingenciesFilePath.empty()) {
return Request::RUN_SIMULATION_SA;
} else {
return Request::RUN_SIMULATION_N;
}
} catch (const DYN::Error& err) {
throw;
} catch (const std::exception& e) {
std::cerr << e.what() << std::endl;
return Request::ERROR;
Expand Down
6 changes: 5 additions & 1 deletion sources/Context.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -350,7 +350,11 @@ void Context::executeSecurityAnalysis() {
multipleJobs->setScenarios(scenarios);
auto saLauncher = boost::make_shared<DYNAlgorithms::SystematicAnalysisLauncher>();
saLauncher->setMultipleJobs(multipleJobs);
saLauncher->setOutputFile("aggregatedResults.xml");
if (def_.outputIsZip) {
saLauncher->setOutputFile("output.zip");
} else {
saLauncher->setOutputFile("aggregatedResults.xml");
}
saLauncher->setDirectory(config_.outputDir().generic_string());
saLauncher->init();
saLauncher->launch();
Expand Down
1 change: 1 addition & 0 deletions sources/Context.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ class Context {
boost::filesystem::path settingFilePath; ///< setting file path for dynamic data base
boost::filesystem::path assemblingFilePath; ///< assembling file path for dynamic data base
boost::filesystem::path contingenciesFilePath; ///< contigencies file path for Security Analysis simulation
bool outputIsZip; ///< true if the output is zip archive, false otherwise
std::string dynawoLogLevel; ///< string representation of the dynawo log level
boost::filesystem::path dynawoResDir; ///< DYNAWO resources
std::string locale; ///< localization
Expand Down
15 changes: 15 additions & 0 deletions sources/main.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@
#include <boost/filesystem.hpp>
#include <boost/property_tree/json_parser.hpp>
#include <boost/property_tree/ptree.hpp>
#include <libzip/ZipFileFactory.h>
#include <libzip/ZipOutputStream.h>
#include <chrono>
#include <cstdlib>
#include <sstream>
Expand Down Expand Up @@ -53,12 +55,14 @@ static inline double elapsed(const std::chrono::steady_clock::time_point &timePo

static boost::shared_ptr<dfl::Context> buildContext(dfl::inputs::SimulationParams const &params, dfl::inputs::Configuration &config) {
auto timeContextStart = std::chrono::steady_clock::now();
bool outputIsZip = !params.runtimeConfig->zipArchive.empty();
dfl::Context::ContextDef def{config.getStartingPointMode(),
params.simulationKind,
params.networkFilePath,
config.settingFilePath(),
config.assemblingFilePath(),
params.runtimeConfig->contingenciesFilePath,
outputIsZip,
params.runtimeConfig->dynawoLogLevel,
params.resourcesDirPath,
params.locale};
Expand Down Expand Up @@ -337,6 +341,17 @@ int main(int argc, char *argv[]) {
boost::shared_ptr<dfl::Context> context = buildContext(params, configSA);
execSimulation(context, params);
}
// std::string archivePath = createAbsolutePath("test.zip", outputDir.generic_string());

// std::map<std::string, std::string > mapData;

// boost::shared_ptr<zip::ZipFile> archive = zip::ZipFileFactory::newInstance();

// for (std::map<std::string, std::string>::const_iterator it = mapData.begin();
// it != mapData.end(); it++) {
// archive->addEntry(it->first, it->second);
// }
// zip::ZipOutputStream::write(archivePath, archive);
} catch (DYN::Error &e) {
if (mpiContext.isRootProc()) {
std::cerr << "Simulation failed: " << e.what() << std::endl;
Expand Down
Loading

0 comments on commit b37b26e

Please sign in to comment.