#810 Add the possibility to zip the inputs and outputs

Signed-off-by: Dimitri Baron <[email protected]>

cmake/TestMain.cmake

@@ -7,14 +7,22 @@
 # SPDX-License-Identifier: MPL-2.0
 #
 
-set(_command ${EXE} --network=res/TestIIDM_${TEST_NAME}.iidm --config=res/config_${TEST_NAME}.json)
+if(${USE_ZIP} STREQUAL "YES")
+  execute_process(COMMAND zip -j res/${TEST_NAME}.zip res/TestIIDM_${TEST_NAME}.iidm res/config_${TEST_NAME}.json)
+  set(_command ${EXE} --network=TestIIDM_${TEST_NAME}.iidm --config=config_${TEST_NAME}.json --input-archive=res/${TEST_NAME}.zip)
+else()
+  set(_command ${EXE} --network=res/TestIIDM_${TEST_NAME}.iidm --config=res/config_${TEST_NAME}.json)
+endif()
 message(STATUS "Execute process: ${_command}")
 execute_process(COMMAND ${_command} RESULT_VARIABLE _result)
 if(_result)
   message(FATAL_ERROR "Execution failed: ${_command}")
 endif()
 
 set(_command ${PYTHON_COMMAND} ${DIFF_SCRIPT} . ${TEST_NAME} res/config_${TEST_NAME}.json)
+if(${USE_ZIP} STREQUAL "YES")
+  list(APPEND _command --output-zip outputs.zip)
+endif()
 if(DEFINED DIFF_SCRIPT)
   message(STATUS "Execute process: ${_command}")
   execute_process(COMMAND ${_command} RESULT_VARIABLE _result)

cmake/TestMainNSA.cmake

@@ -7,7 +7,12 @@
 # SPDX-License-Identifier: MPL-2.0
 #
 
-set(_dfl_cmd ${EXE} --network=res/TestIIDM_${TEST_NAME}.iidm --config=res/config_${TEST_NAME}.json --contingencies=res/contingencies_${TEST_NAME}.json --nsa)
+if(${USE_ZIP} STREQUAL "YES")
+  execute_process(COMMAND zip -j res/${TEST_NAME}.zip res/TestIIDM_${TEST_NAME}.iidm res/config_${TEST_NAME}.json res/contingencies_${TEST_NAME}.json)
+  set(_dfl_cmd ${EXE} --network=TestIIDM_${TEST_NAME}.iidm --config=config_${TEST_NAME}.json --contingencies=contingencies_${TEST_NAME}.json --input-archive=res/${TEST_NAME}.zip --nsa)
+else()
+  set(_dfl_cmd ${EXE} --network=res/TestIIDM_${TEST_NAME}.iidm --config=res/config_${TEST_NAME}.json --contingencies=res/contingencies_${TEST_NAME}.json --nsa)
+endif()
 if(NOT DEFINED USE_MPI OR USE_MPI STREQUAL "")
   message(FATAL_ERROR "USE_MPI is not defined")
 endif()
@@ -26,6 +31,9 @@ if(_result)
 endif()
 
 set(_command ${PYTHON_COMMAND} ${N_DIFF_SCRIPT} . ${TEST_NAME} res/config_${TEST_NAME}.json)
+if(${USE_ZIP} STREQUAL "YES")
+  list(APPEND _command --output-zip outputs.zip)
+endif()
 if(DEFINED N_DIFF_SCRIPT)
   message(STATUS "Execute process: ${_command}")
   execute_process(COMMAND ${_command} RESULT_VARIABLE _result)

cmake/TestMainSA.cmake

@@ -7,7 +7,12 @@
 # SPDX-License-Identifier: MPL-2.0
 #
 
-set(_dfl_cmd ${EXE} --network=res/TestIIDM_${TEST_NAME}.iidm --config=res/config_${TEST_NAME}.json --contingencies=res/contingencies_${TEST_NAME}.json)
+if(${USE_ZIP} STREQUAL "YES")
+  execute_process(COMMAND zip -j res/${TEST_NAME}.zip res/TestIIDM_${TEST_NAME}.iidm res/config_${TEST_NAME}.json res/contingencies_${TEST_NAME}.json)
+  set(_dfl_cmd ${EXE} --network=TestIIDM_${TEST_NAME}.iidm --config=config_${TEST_NAME}.json --contingencies=contingencies_${TEST_NAME}.json --input-archive=res/${TEST_NAME}.zip)
+else ()
+  set(_dfl_cmd ${EXE} --network=res/TestIIDM_${TEST_NAME}.iidm --config=res/config_${TEST_NAME}.json --contingencies=res/contingencies_${TEST_NAME}.json)
+endif()
 if(NOT DEFINED USE_MPI OR USE_MPI STREQUAL "")
   message(FATAL_ERROR "USE_MPI is not defined")
 endif()
@@ -26,6 +31,9 @@ if(_result)
 endif()
 
 set(_command ${PYTHON_COMMAND} ${DIFF_SCRIPT} . ${TEST_NAME} res/config_${TEST_NAME}.json)
+if(${USE_ZIP} STREQUAL "YES")
+  list(APPEND _command --output-zip outputs.zip)
+endif()
 if(DEFINED DIFF_SCRIPT)
   message(STATUS "Execute process: ${_command}")
   execute_process(COMMAND ${_command} RESULT_VARIABLE _result)

documentation/configuringDynaflowLauncher/DynaflowLaunchersInputFiles.tex

@@ -40,17 +40,23 @@ \subsubsection{Dynaflow}
 The command line used to launch Dynaflow is the following:
 
 \begin{lstlisting}[language=bash, breaklines=true, breakatwhitespace=false, columns=fullflexible]
-$> ./myEnvDFL.sh <PATH TO NETWORK FILE> <PATH TO CONFIG FILE>
+$> ./myEnvDFL.sh launch <PATH TO NETWORK FILE> <PATH TO CONFIG FILE>
 \end{lstlisting}
 
 for example:
 
 \begin{lstlisting}[language=bash, breaklines=true, breakatwhitespace=false, columns=fullflexible]
-$> ./myEnvDFL.sh tests/main/res/TestIIDM_launch.iidm tests/main/res/config_launch.json
+$> ./myEnvDFL.sh launch tests/main/res/TestIIDM_launch.iidm tests/main/res/config_launch.json
 \end{lstlisting}
 
 This command will compute the steady-state of the network and provides as
-output the constraints and lostEquipments files (see \Dynawo documentation for more details on those files).
+output the constraints and lostEquipments files (see \Dynawo documentation for more details on those files).\newline
+
+You can also provide a zip archive as input. In this case, you need to specify the path of the zip archive along with the paths of the unzipped files :
+
+\begin{lstlisting}[language=bash, breaklines=true, breakatwhitespace=false, columns=fullflexible]
+$> ./myEnvDFL.sh launch tests/main/res/TestIIDM_launch.iidm tests/main/res/config_launch.json --input-archive tests/main/archive.zip
+\end{lstlisting}
 
 \subsubsection{Dynaflow systematic analysis}
 
@@ -66,13 +72,13 @@ \subsubsection{Dynaflow systematic analysis}
 The command line used to launch Dynaflow static analysis is the following:
 
 \begin{lstlisting}[language=bash, breaklines=true, breakatwhitespace=false, columns=fullflexible]
-$> ./myEnvDFL.sh <PATH TO NETWORK FILE> <PATH TO CONFIG FILE> <PATH TO CONTINGENCIES FILE>
+$> ./myEnvDFL.sh launch-sa <PATH TO NETWORK FILE> <PATH TO CONFIG FILE> <PATH TO CONTINGENCIES FILE>
 \end{lstlisting}
 
 for example:
 
 \begin{lstlisting}[language=bash, breaklines=true, breakatwhitespace=false, columns=fullflexible]
-$> ./myEnvDFL.sh tests/main_sa/res/TestIIDM_launch.iidm tests/main_sa/res/config_launch.json tests/main_sa/res/contingencies_launch.json
+$> ./myEnvDFL.sh launch-sa tests/main_sa/res/TestIIDM_launch.iidm tests/main_sa/res/config_launch.json tests/main_sa/res/contingencies_launch.json
 \end{lstlisting}
 
 This command will simulate all the events described in the contingencies file and provide as outputs the constraints and lostEquipments files for each of them.
@@ -87,7 +93,7 @@ \subsubsection{Dynaflow steady-state and security analysis workflow}
 For example:
 
 \begin{lstlisting}[language=bash, breaklines=true, breakatwhitespace=false, columns=fullflexible]
-  $> ./myEnvDFL.sh tests/main_sa/res/TestIIDM_launch.iidm tests/main_sa/res/config_launch.json tests/main_sa/res/contingencies_launch.json --nsa
+  $> ./myEnvDFL.sh launch-nsa tests/main_sa/res/TestIIDM_launch.iidm tests/main_sa/res/config_launch.json tests/main_sa/res/contingencies_launch.json
 \end{lstlisting}
 
 \subsection[Dynaflow-launcher outputs]{Dynaflow-launcher outputs}

etc/Dictionaries/DFLError_en_GB.dic

@@ -33,7 +33,8 @@ MissingICInWarmStartingPointMode =  no initial condition (p, q, v or theta) was
 
 //------------------ Outputs -----------------------
 MissingGeneratorHvdcParameterInSettings = parameter %1% not found in settings file for generator or hvdc %2%
-NoSVCInFlatStartingPointMode =        simulation of secondary voltage controls is not possible with 'FLAT' starting point mode
+NoSVCInFlatStartingPointMode =     simulation of secondary voltage controls is not possible with 'FLAT' starting point mode
+FileCreationFailed           =     failed to create %1%
 //------------------ Main ---------------------------
 EnvVariableMissing           =     cannot find environnement variable %1% please check runtime environment
 NetworkFileNotFound          =     network file %1% does not exist
@@ -42,3 +43,4 @@ ContextProcessError          =     processing of the network %1% failed
 ConnexityError               =     slack node of id %1% not present in main connex component
 NetworkHasNoRegulatingGenerator         =     network file %1% has no generator regulating voltage in the main connex component
 SlackNodeNotFound            =     no slack node found for network '%1%'
+FailToOpenLogFile            =     failed to open %1% log file

etc/warnings_to_filter.txt

@@ -4,6 +4,7 @@ warning: return type of member dfl::inputs::helper::updateValue is not documente
 warning: return type of member dfl::inputs::helper::parserFile is not documented
 warning: return type of member dfl::common::validate is not documented
 warning: return type of member dfl::common::Log::init is not documented
+warning: return type of member dfl::common::Log::addLogFileContentInMapData is not documented
 warning: return type of member dfl::inputs::ContingencyElement::isCompatible is not documented
 warning: return type of member dfl::inputs::NetworkManager::updateMapRegulatingBuses is not documented
 warning: return type of member dfl::inputs::NetworkManager::updateVSCMapRegulatingBuses is not documented

scripts/diff.py

@@ -13,6 +13,7 @@
 import os
 import sys
 import argparse
+import zipfile
 import json
 try:
     from itertools import zip_longest
@@ -29,11 +30,12 @@
 def get_argparser():
     parser = argparse.ArgumentParser()
 
-    parser.add_argument("--verbose", "-v",
-                        help="Print comparing result", action="store_true")
     parser.add_argument("root", type=str, help="Root directory to process")
     parser.add_argument("testdir", type=str, help="Test directory to process")
     parser.add_argument("config", type=str, help="Simulation configuration file")
+    parser.add_argument("--verbose", "-v",
+                        help="Print comparing result", action="store_true")
+    parser.add_argument("--output-zip", type=str, help="zip archive to process")
 
     return parser
 
@@ -54,82 +56,89 @@ def get_argparser():
             except KeyError:
                 pass
 
+    if options.output_zip:
+        zip_archive_parent_directory = os.path.join(options.root, "resultsTestsTmp", options.testdir)
+        zip_archive_path = os.path.join(zip_archive_parent_directory, options.output_zip)
+        unzipped_archive_path = zip_archive_parent_directory
+        try:
+            with zipfile.ZipFile(zip_archive_path, "r") as zip_ref:
+                zip_ref.extractall(unzipped_archive_path)
+        except FileNotFoundError:
+            print("[ERROR] " + zip_archive_path + " is missing")
+            nb_differences += 1
+
     #output IIDM
     if buildType == "Debug" or buildType == "Release" or buildType == "Coverage":
 
         is_constraints_generated = (buildType == "Debug" or (buildType == "Release" and "CONSTRAINTS" in chosen_outputs))
         is_lost_equipement_generated = (buildType == "Debug" or (buildType == "Release" and "LOSTEQ" in chosen_outputs))
 
-        result_path = os.path.realpath(os.path.join(
-            options.root, "resultsTestsTmp", options.testdir, "outputs/finalState/outputIIDM.xml"))
-        reference_path = os.path.realpath(os.path.join(
-            options.root, "reference", options.testdir, "outputIIDM.xml"))
+        results_root = os.path.realpath(os.path.join(options.root, "resultsTestsTmp", options.testdir))
+        reference_root = os.path.realpath(os.path.join(options.root, "reference", options.testdir))
 
-        if not os.path.exists(reference_path):
-            print("Reference path " + reference_path +
-                " does not exist : not checked")
+        outputiidm_result_paths = list()
+        outputiidm_result_paths.append(os.path.realpath(os.path.join(results_root, "outputs/finalState/outputIIDM.xml")))
+        outputiidm_reference_path = os.path.realpath(os.path.join(reference_root, "outputIIDM.xml"))
+
+        if not os.path.exists(outputiidm_reference_path):
+            print("Reference path " + outputiidm_reference_path + " does not exist : not checked")
         else:
-            if options.verbose:
-                print("comparing " + result_path + " and " + reference_path)
+            for outputiidm_result_path in outputiidm_result_paths:
+                if options.verbose:
+                    print("comparing " + outputiidm_result_path + " and " + outputiidm_reference_path)
 
-            (nb_differences_local, msg) = iidmDiff.OutputIIDMCloseEnough(
-                result_path, reference_path)
-            if nb_differences_local > 0:
-                print("[ERROR] " + result_path + ": " + msg)
-            elif options.verbose:
-                print("No difference")
-            nb_differences += nb_differences_local
+                (nb_differences_local, msg) = iidmDiff.OutputIIDMCloseEnough(outputiidm_result_path, outputiidm_reference_path)
+                if nb_differences_local > 0:
+                    print("[ERROR] " + outputiidm_result_path + ": " + msg)
+                elif options.verbose:
+                    print("No difference")
+                nb_differences += nb_differences_local
 
         # dyd and par
-        results_root = os.path.realpath(os.path.join(options.root, "resultsTestsTmp", options.testdir))
-        reference_root = os.path.realpath(os.path.join(options.root, "reference", options.testdir))
         nb_differences += compare_dyd_and_par_files(results_root, reference_root, options.verbose)
 
         if is_constraints_generated:
             #constraints
-            result_path = os.path.realpath(os.path.join(
-                options.root, "resultsTestsTmp", options.testdir, "outputs/constraints/constraints.xml"))
-            reference_path = os.path.realpath(os.path.join(
-                options.root, "reference", options.testdir, "constraints.xml"))
-
-            if not os.path.exists(reference_path):
-                print("Reference path " + reference_path +
-                        " does not exist : not checked")
-            else:
-                if options.verbose:
-                    print("comparing " + result_path + " and " + reference_path)
+            constraints_result_paths = list()
+            constraints_result_paths.append(os.path.realpath(os.path.join(results_root, "outputs/constraints/constraints.xml")))
+            constraints_reference_path = os.path.realpath(os.path.join(reference_root, "constraints.xml"))
 
-                (nb_differences_local, msg) = constraintsDiff.output_constraints_close_enough(
-                    result_path, reference_path)
-                if nb_differences_local > 0:
-                    print("[ERROR] constraints file " + result_path + " different from reference file " + reference_path)
-                    print(msg)
-                elif options.verbose:
-                    print("No difference")
-                nb_differences += nb_differences_local
+            if not os.path.exists(constraints_reference_path):
+                print("Reference path " + constraints_reference_path + " does not exist : not checked")
+            else:
+                for constraints_result_path in constraints_result_paths:
+                    if options.verbose:
+                        print("comparing " + constraints_result_path + " and " + constraints_reference_path)
+
+                    (nb_differences_local, msg) = constraintsDiff.output_constraints_close_enough(constraints_result_path, constraints_reference_path)
+                    if nb_differences_local > 0:
+                        print("[ERROR] constraints file " + constraints_result_path + " different from reference file " + constraints_reference_path)
+                        print(msg)
+                    elif options.verbose:
+                        print("No difference")
+                    nb_differences += nb_differences_local
 
         if is_lost_equipement_generated:
             #lost equipments
-            result_path = os.path.realpath(os.path.join(
-                options.root, "resultsTestsTmp", options.testdir, "outputs/lostEquipments/lostEquipments.xml"))
-            reference_path = os.path.realpath(os.path.join(
-                options.root, "reference", options.testdir, "lostEquipments.xml"))
-
-            if not os.path.exists(reference_path):
-                print("Reference path " + reference_path +
-                        " does not exist : not checked")
-            else:
-                if options.verbose:
-                    print("comparing " + result_path + " and " + reference_path)
+            lost_equipments_result_paths = list()
+            lost_equipments_result_paths.append(os.path.realpath(os.path.join(results_root, "outputs/lostEquipments/lostEquipments.xml")))
+            lost_equipments_reference_path = os.path.realpath(os.path.join(reference_root, "lostEquipments.xml"))
 
-                # compare line per line with universal newline, stop at first diff
-                with open(result_path) as f1, open(reference_path) as f2:
-                    identical = all(l1 == l2 for l1, l2 in zip_longest(f1, f2))
-                if identical:
-                    print("No difference")
-                else:
-                    print("[ERROR] lost equipments file " + result_path + " different from reference file " + reference_path)
-                    nb_differences += 1
+            if not os.path.exists(lost_equipments_reference_path):
+                print("Reference path " + lost_equipments_reference_path + " does not exist : not checked")
+            else:
+                for lost_equipments_result_path in lost_equipments_result_paths:
+                    if options.verbose:
+                        print("comparing " + lost_equipments_result_path + " and " + lost_equipments_reference_path)
+
+                    # compare line per line with universal newline, stop at first diff
+                    with open(lost_equipments_result_path) as f1, open(lost_equipments_reference_path) as f2:
+                        identical = all(l1 == l2 for l1, l2 in zip_longest(f1, f2))
+                    if identical:
+                        print("No difference")
+                    else:
+                        print("[ERROR] lost equipments file " + lost_equipments_result_path + " different from reference file " + lost_equipments_reference_path)
+                        nb_differences += 1
     else:
         raise UnknownBuildType(buildType)
 

scripts/diffContingencies.py

@@ -15,6 +15,7 @@
 import argparse
 import json
 import iidmDiff
+import zipfile
 import constraintsDiff
 try:
     from itertools import zip_longest
@@ -34,6 +35,7 @@ def get_argparser():
     parser.add_argument("root", type=str, help="Root directory to process")
     parser.add_argument("testdir", type=str, help="Test directory to process")
     parser.add_argument("config", type=str, help="Simulation configuration file")
+    parser.add_argument("--output-zip", type=str, help="zip archive to process")
 
     return parser
 
@@ -46,6 +48,17 @@ def compare_file(options, contingency_folder, chosen_outputs):
 
     if buildType == "Debug" or buildType == "Release" or buildType == "Coverage":
 
+        if options.output_zip:
+            zip_archive_parent_directory = os.path.join(options.root, "resultsTestsTmp", options.testdir)
+            zip_archive_path = os.path.join(zip_archive_parent_directory, options.output_zip)
+            unzipped_archive_path = zip_archive_parent_directory
+            try:
+                with zipfile.ZipFile(zip_archive_path, "r") as zip_ref:
+                    zip_ref.extractall(unzipped_archive_path)
+            except FileNotFoundError:
+                print("[ERROR] " + zip_archive_path + " is missing")
+                nb_differences += 1
+
         if buildType == "Debug" or (buildType == "Release" and "STEADYSTATE" in chosen_outputs) or buildType == "Coverage":
             # output IIDM
             result_path = full_path(