Fix generation of weights in API mode [ANT-2196] (#944)

When generating weights.txt, instead of using list of MPS files generated by simulator in the archive, use the list of MPS files generated by XPANSION in output_dir/lp. This makes it compatible with API mode. Also added a cucumber test with a new step to launch xpansion (and fixed the "simulation succeeds" step) --------- Co-authored-by: Jason Maréchal <[email protected]>
AntaresSimulatorTeam · Oct 11, 2024 · bbf7c19 · bbf7c19
1 parent f032ccc
commit bbf7c19
Show file tree

Hide file tree

Showing 8 changed files with 109 additions and 42 deletions.
diff --git a/src/cpp/lpnamer/input_reader/WeightsFileWriter.cpp b/src/cpp/lpnamer/input_reader/WeightsFileWriter.cpp
@@ -2,18 +2,16 @@
 
 #include <fstream>
 #include <numeric>
+#include <ranges>
 
-#include "ArchiveReader.h"
 #include "StringManip.h"
 
 YearlyWeightsWriter::YearlyWeightsWriter(
     const std::filesystem::path& xpansion_output_dir,
-    const std::filesystem::path& antares_archive_path,
     const std::vector<double>& weights_vector,
     const std::filesystem::path& output_file,
     const std::vector<int>& active_years)
     : xpansion_output_dir_(xpansion_output_dir),
-      antares_archive_path_(antares_archive_path),
       weights_vector_(weights_vector),
       output_file_(output_file),
       active_years_(active_years) {
@@ -30,19 +28,17 @@ void YearlyWeightsWriter::CreateWeightFile() {
 
 void YearlyWeightsWriter::FillMpsWeightsMap() {
   mps_weights_.clear();
-  auto zip_reader = ArchiveReader(antares_archive_path_);
-  zip_reader.Open();
-  zip_reader.LoadEntriesPath();
-  const auto& mps_files = zip_reader.GetEntriesPathWithExtension(".mps");
-  for (auto& mps_file : mps_files) {
-    auto year = GetYearFromMpsName(mps_file.string());
+  auto it = std::filesystem::directory_iterator(xpansion_lp_dir_);
+  auto mps_list = it | std::views::filter([](const auto& p) {
+                    return p.path().extension() == ".mps";
+                  });
+  for (auto& mps_file : mps_list ) {
+    auto year = GetYearFromMpsName(mps_file.path().filename().string());
     auto year_index =
         std::find(active_years_.begin(), active_years_.end(), year) -
         active_years_.begin();
-    mps_weights_[mps_file.filename()] = weights_vector_[year_index];
+    mps_weights_[mps_file.path().filename()] = weights_vector_[year_index];
   }
-  zip_reader.Close();
-  zip_reader.Delete();
 }
 
 int YearlyWeightsWriter::GetYearFromMpsName(

diff --git a/src/cpp/lpnamer/input_reader/WeightsFileWriter.h b/src/cpp/lpnamer/input_reader/WeightsFileWriter.h
@@ -7,7 +7,6 @@
 class YearlyWeightsWriter {
  public:
   explicit YearlyWeightsWriter(const std::filesystem::path& xpansion_output_dir,
-                               const std::filesystem::path& zipped_output_path,
                                const std::vector<double>& weights_vector,
                                const std::filesystem::path& output_file,
                                const std::vector<int>& active_years);

diff --git a/src/cpp/lpnamer/main/ProblemGeneration.cpp b/src/cpp/lpnamer/main/ProblemGeneration.cpp
@@ -2,7 +2,6 @@
 #include "include/ProblemGeneration.h"
 
 #include <antares/api/solver.h>
-#include <antares/api/SimulationResults.h>
 
 #include <execution>
 #include <iostream>
@@ -24,7 +23,6 @@
 #include "ProblemVariablesFileAdapter.h"
 #include "ProblemVariablesFromProblemAdapter.h"
 #include "ProblemVariablesZipAdapter.h"
-#include "StringManip.h"
 #include "Timer.h"
 #include "Version.h"
 #include "WeightsFileReader.h"
@@ -62,9 +60,9 @@ ProblemGeneration::ProblemGeneration(ProblemGenerationOptions& options)
 
 std::filesystem::path ProblemGeneration::performAntaresSimulation() {
   auto results = Antares::API::PerformSimulation(options_.StudyPath());
-  //Add parallel
+  // Add parallel
 
-  //Handle errors
+  // Handle errors
   if (results.error) {
     throw LogUtils::XpansionError<std::runtime_error>(
         "Antares simulation failed:\n\t" + results.error->reason, LOGLOCATION);
@@ -94,7 +92,8 @@ std::filesystem::path ProblemGeneration::updateProblems() {
     // antares
   }
 
-  if (mode_ == SimulationInputMode::ANTARES_API || mode_ == SimulationInputMode::FILE) {
+  if (mode_ == SimulationInputMode::ANTARES_API ||
+      mode_ == SimulationInputMode::FILE) {
     xpansion_output_dir = simulation_dir_;
   }
 
@@ -121,7 +120,6 @@ std::shared_ptr<ArchiveReader> InstantiateZipReader(
     const std::filesystem::path& antares_archive_path);
 void ProblemGeneration::ProcessWeights(
     const std::filesystem::path& xpansion_output_dir,
-    const std::filesystem::path& antares_archive_path,
     const std::filesystem::path& weights_file,
     ProblemGenerationLog::ProblemGenerationLoggerSharedPointer logger) {
   const auto settings_dir = xpansion_output_dir / ".." / ".." / "settings";
@@ -132,9 +130,9 @@ void ProblemGeneration::ProcessWeights(
                                         logger);
   weights_file_reader.CheckWeightsFile();
   auto weights_vector = weights_file_reader.WeightsList();
-  auto yearly_weight_writer = YearlyWeightsWriter(
-      xpansion_output_dir, antares_archive_path, weights_vector,
-      weights_file.filename(), active_years);
+  auto yearly_weight_writer =
+      YearlyWeightsWriter(xpansion_output_dir, weights_vector,
+                          weights_file.filename(), active_years);
   yearly_weight_writer.CreateWeightFile();
 }
 
@@ -181,7 +179,8 @@ void validateMasterFormulation(
  * @param solver_name
  * @param mpsList
  * @param lpDir_
- * @param reader shared pointer to the archive reader to share with ZipProblemsProviderAdapter
+ * @param reader shared pointer to the archive reader to share with
+ * ZipProblemsProviderAdapter
  * @param with_archive
  * @param lps data from antares. Passed by reference to prevent heavy copy
  * @return
@@ -201,8 +200,8 @@ std::vector<std::shared_ptr<Problem>> ProblemGeneration::getXpansionProblems(
       return adapter.provideProblems(solver_name, solver_log_manager);
     }
     case SimulationInputMode::ARCHIVE: {
-      ZipProblemsProviderAdapter adapter(
-          lpDir_, std::move(reader), problem_names);
+      ZipProblemsProviderAdapter adapter(lpDir_, std::move(reader),
+                                         problem_names);
       return adapter.provideProblems(solver_name, solver_log_manager);
     }
     case SimulationInputMode::ANTARES_API: {
@@ -231,10 +230,6 @@ void ProblemGeneration::RunProblemGeneration(
   SolverLoader::GetAvailableSolvers(logger);  // Dirty fix to populate static
                                               // value outside multi thread code
   Timer problem_generation_timer;
-  if (!weights_file.empty()) {
-    ProcessWeights(xpansion_output_dir, antares_archive_path, weights_file,
-                   logger);
-  }
 
   ExtractUtilsFiles(antares_archive_path, xpansion_output_dir, logger);
 
@@ -256,7 +251,6 @@ void ProblemGeneration::RunProblemGeneration(
   (*logger)(LogUtils::LOGLEVEL::INFO)
       << "rename problems: " << std::boolalpha << rename_problems << std::endl;
 
-
   auto files_mapper = FilesMapper(antares_archive_path, xpansion_output_dir);
   auto mpsList = files_mapper.MpsAndVariablesFilesVect();
 
@@ -318,6 +312,10 @@ void ProblemGeneration::RunProblemGeneration(
                                     mps_file_writer.get());
       });
 
+  if (!weights_file.empty()) {
+    ProcessWeights(xpansion_output_dir, weights_file, logger);
+  }
+
   if (mode_ == SimulationInputMode::ARCHIVE) {
     reader->Close();
     reader->Delete();

diff --git a/src/cpp/lpnamer/main/include/ProblemGeneration.h b/src/cpp/lpnamer/main/include/ProblemGeneration.h
@@ -38,7 +38,6 @@ class ProblemGeneration {
 
   void ProcessWeights(
       const std::filesystem::path& xpansion_output_dir,
-      const std::filesystem::path& antares_archive_path,
       const std::filesystem::path& weights_file,
       ProblemGenerationLog::ProblemGenerationLoggerSharedPointer logger);
   void ExtractUtilsFiles(

diff --git a/tests/cpp/lp_namer/CMakeLists.txt b/tests/cpp/lp_namer/CMakeLists.txt
@@ -34,6 +34,7 @@ add_executable (lp_namer_tests
         LoggerBuilder.cpp
         ProblemGenerationLoggerTest.cpp
         WeightsFileReaderTest.cpp
+        WeightsFileWriterTest.cpp
         LpFilesExtractorTest.cpp
         MpsTxtWriterTest.cpp
         AntaresProblemToXpansionProblemTranslatorTest.cpp

diff --git a/tests/cpp/lp_namer/WeightsFileWriterTest.cpp b/tests/cpp/lp_namer/WeightsFileWriterTest.cpp
@@ -0,0 +1,45 @@
+#include <fstream>
+
+#include "WeightsFileWriter.h"
+#include "gtest/gtest.h"
+
+void writeDummyFileInTempLpDir(std::string name) {
+  auto tmp_path = std::filesystem::temp_directory_path() / "lp" / name;
+  std::ofstream writer;
+  writer.open(tmp_path);
+  writer << std::endl;
+  writer.close();
+}
+
+TEST(WeightsFileWriterTest, CorrectlyWriteWeightsFile) {
+  auto tempDir = std::filesystem::temp_directory_path() / "lp";
+  if (!std::filesystem::is_directory(tempDir)) {
+    std::filesystem::create_directory(tempDir);
+  }
+  writeDummyFileInTempLpDir("problem-1-1--optim-nb-1.mps");
+  writeDummyFileInTempLpDir("problem-1-50--optim-nb-1.mps");
+  writeDummyFileInTempLpDir("problem-2-10--optim-nb-1.mps");
+  writeDummyFileInTempLpDir("problem-2-11--optim-nb-1.mps");
+  writeDummyFileInTempLpDir("problem-2-30--optim-nb-1.mps");
+  writeDummyFileInTempLpDir("problem-3-20--optim-nb-1.mps");
+  writeDummyFileInTempLpDir("problem-3-30--optim-nb-1.txt");
+
+  auto yearly_weight_writer =
+      YearlyWeightsWriter(std::filesystem::temp_directory_path(), {3, 5, 7},
+                          "weights_123.txt", {1, 2, 3});
+  yearly_weight_writer.CreateWeightFile();
+
+  std::ifstream reader(tempDir / "weights_123.txt");
+  std::string actual((std::istreambuf_iterator<char>(reader)),
+                     std::istreambuf_iterator<char>());
+  std::string expected = R"xxx(problem-1-1--optim-nb-1.mps 3
+problem-1-50--optim-nb-1.mps 3
+problem-2-10--optim-nb-1.mps 5
+problem-2-11--optim-nb-1.mps 5
+problem-2-30--optim-nb-1.mps 5
+problem-3-20--optim-nb-1.mps 7
+WEIGHT_SUM 15
+)xxx";
+
+  EXPECT_EQ(expected, actual);
+}
diff --git a/tests/end_to_end/cucumber/features/weights.feature b/tests/end_to_end/cucumber/features/weights.feature
@@ -0,0 +1,9 @@
+Feature: add weights on MC years
+
+  @slow @short @full-launch
+  Scenario: handling different weights on MC years in API mode
+    # For now, only check that the simulation succeeds
+    # TODO : add more non-regression tests when we have more steps
+    Given the study path is "examples/SmallTestFiveCandidatesWithWeights" for a study with weights
+    When I run antares-xpansion in memory with the benders method and 1 proc(s)
+    Then the simulation succeeds
diff --git a/tests/end_to_end/cucumber/steps/steps.py b/tests/end_to_end/cucumber/steps/steps.py
@@ -15,14 +15,20 @@ def study_path_is(context, string):
                                       string.replace("/", os.sep))
 
 
-def build_outer_loop_command(context):
-    command = get_mpi_command(allow_run_as_root=context.allow_run_as_root, nproc=context.nproc)
+def build_outer_loop_command(context, n: int):
+    command = get_mpi_command(allow_run_as_root=context.allow_run_as_root, nproc=n)
     exe_path = Path(get_conf("DEFAULT_INSTALL_DIR")) / get_conf("OUTER_LOOP")
     command.append(str(exe_path))
     command.append("options.json")
     return command
 
 
+def build_launch_command(study_dir: str, method: str, nproc: int, in_memory: bool):
+    command = f"python ../../src/python/launch.py --installDir {get_conf('DEFAULT_INSTALL_DIR')} --dataDir {study_dir} --method {method} -n {nproc} --oversubscribe"
+    if in_memory:
+        command += " --memory"
+    return command
+
 
 def read_outputs(output_path):
     with open(output_path, 'r') as file:
@@ -31,11 +37,10 @@ def read_outputs(output_path):
     return outputs
 
 
-@when('I run outer loop with {n} proc(s)')
+@when('I run outer loop with {n:d} proc(s)')
 def run_outer_loop(context, n):
-    context.nproc = int(n)
     context.allow_run_as_root = get_conf("allow_run_as_root")
-    command = build_outer_loop_command(context)
+    command = build_outer_loop_command(context, n)
     print(f"Running command: {command}")
     old_cwd = os.getcwd()
     lp_path = Path(context.study_path) / "lp"
@@ -51,20 +56,28 @@ def run_outer_loop(context, n):
     os.chdir(old_cwd)
 
 
-@then("the simulation takes less than {seconds} seconds")
+@when('I run antares-xpansion in memory with the {method} method and {n:d} proc(s)')
+def run_antares_xpansion(context, method, n):
+    command = build_launch_command(context.study_path, method, n, True)
+    process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, shell=True)
+    out, err = process.communicate()
+    context.return_code = process.returncode
+    context.outputs = read_outputs(Path(get_results_file_path_from_logs(out)))
+
+
+@then("the simulation takes less than {seconds:d} seconds")
 def check_simu_time(context, seconds):
-    assert context.outputs["run_duration"] <= float(seconds)
+    assert context.outputs["run_duration"] <= seconds
 
 
 @then("the simulation succeeds")
 def simu_success(context):
-    return context.return_code == 0
+    assert context.return_code == 0
 
 
-@then("the expected overall cost is {value}")
+@then("the expected overall cost is {value:g}")
 def check_overall_cost(context, value):
-    np.testing.assert_allclose(float(value),
-                               context.outputs["solution"]["overall_cost"], rtol=1e-6, atol=0)
+    np.testing.assert_allclose(value, context.outputs["solution"]["overall_cost"], rtol=1e-6, atol=0)
 
 
 def assert_dict_allclose(actual, expected, rtol=1e-06, atol=0):
@@ -82,3 +95,10 @@ def assert_dict_allclose(actual, expected, rtol=1e-06, atol=0):
 def check_solution(context):
     expected_solution = {row['variable']: float(row['value']) for row in context.table}
     assert_dict_allclose(context.outputs["solution"]["values"], expected_solution)
+
+
+def get_results_file_path_from_logs(logs: bytes) -> str:
+    for line in logs.splitlines():
+        if b'Optimization results available in : ' in line:
+            return line.split(b'Optimization results available in : ')[1].decode('ascii')
+    raise LookupError("Could not find results file path in output logs")