Merge pull request #208 from rl-institut/fix/add_missing_input_files

add missing input files to output

simba/__main__.py

@@ -26,18 +26,9 @@
         except NotADirectoryError:
             # can't create new directory (may be write protected): no output
             args.output_directory = None
-    if args.output_directory is not None:
-        # copy input files to output to ensure reproducibility
-        copy_list = [args.config, args.electrified_stations_path, args.vehicle_types_path]
-        if "station_optimization" in args.mode:
-            copy_list.append(args.optimizer_config)
-
-        # only copy cost params if they exist
-        if args.cost_parameters_path is not None:
-            copy_list.append(args.cost_parameters_path)
-        for c_file in map(Path, copy_list):
-            shutil.copy(c_file, args.output_directory_input / c_file.name)
 
+        # copy basic input to output to ensure reproducibility
+        util.save_input_file(args.config, args)
         util.save_version(args.output_directory_input / "program_version.txt")
 
     util.setup_logging(args, time_str)
@@ -49,3 +40,9 @@
         raise
     finally:
         logging.shutdown()
+        if args.zip_output and args.output_directory is not None and args.output_directory.exists():
+            # compress output directory after simulation
+            # generate <output_directory_name>.zip at location of original output directory
+            shutil.make_archive(args.output_directory, 'zip', args.output_directory)
+            # remove original output directory
+            shutil.rmtree(args.output_directory)

simba/data_container.py

@@ -31,6 +31,8 @@ def __init__(self):
         # List of trip dictionaries containing trip information like arrival time and station
         # departure time and station, distance and more
         self.trip_data: [dict] = []
+        # Simulation arguments
+        self.args = None
 
     def fill_with_args(self, args: argparse.Namespace) -> 'DataContainer':
         """ Fill DataContainer with data from file_paths defined in args.
@@ -40,6 +42,7 @@ def fill_with_args(self, args: argparse.Namespace) -> 'DataContainer':
             level_of_loading_over_day_path, station_data_path
         :return: self
         """
+        self.args = args
 
         return self.fill_with_paths(
             trips_file_path=args.input_schedule,
@@ -69,6 +72,7 @@ def add_trip_data_from_csv(self, file_path: Path) -> 'DataContainer':
                 trip_d[TEMPERATURE] = util.cast_float_or_none(trip.get(TEMPERATURE))
                 trip_d["distance"] = float(trip["distance"])
                 self.trip_data.append(trip_d)
+        util.save_input_file(file_path, self.args)
         return self
 
     def add_station_geo_data(self, data: dict) -> None:
@@ -162,6 +166,7 @@ def add_station_geo_data_from_csv(self, file_path: Path) -> 'DataContainer':
                 msg=f"Can't parse numeric data in line {line_num + 2} from file {file_path}.",
                 level=100)
             raise
+        util.save_input_file(file_path, self.args)
         return self
 
     def add_level_of_loading_data(self, data: dict) -> 'DataContainer':
@@ -186,6 +191,7 @@ def add_level_of_loading_data_from_csv(self, file_path: Path) -> 'DataContainer'
         index = "hour"
         column = "level_of_loading"
         level_of_loading_data_dict = util.get_dict_from_csv(column, file_path, index)
+        util.save_input_file(file_path, self.args)
         self.add_level_of_loading_data(level_of_loading_data_dict)
         return self
 
@@ -209,6 +215,7 @@ def add_temperature_data_from_csv(self, file_path: Path) -> 'DataContainer':
         index = "hour"
         column = "temperature"
         temperature_data_dict = util.get_dict_from_csv(column, file_path, index)
+        util.save_input_file(file_path, self.args)
         self.add_temperature_data(temperature_data_dict)
         return self
 
@@ -231,6 +238,7 @@ def add_cost_parameters_from_json(self, file_path: Path) -> 'DataContainer':
         :return: DataContainer containing cost parameters
         """
         cost_parameters = self.get_json_from_file(file_path, "cost parameters")
+        util.save_input_file(file_path, self.args)
         self.add_cost_parameters(cost_parameters)
         return self
 
@@ -253,6 +261,7 @@ def add_stations_from_json(self, file_path: Path) -> 'DataContainer':
 
         """
         stations = self.get_json_from_file(file_path, "electrified stations")
+        util.save_input_file(file_path, self.args)
         self.add_stations(stations)
         return self
 
@@ -274,6 +283,7 @@ def add_vehicle_types_from_json(self, file_path: Path):
         :return: DataContainer containing vehicle types
         """
         vehicle_types = self.get_json_from_file(file_path, "vehicle types")
+        util.save_input_file(file_path, self.args)
         self.add_vehicle_types(vehicle_types)
         return self
 
@@ -311,6 +321,7 @@ def add_consumption_data_from_vehicle_type_linked_files(self):
             delim = util.get_csv_delim(mileage_path)
             df = pd.read_csv(mileage_path, sep=delim)
             self.add_consumption_data(mileage_path, df)
+            util.save_input_file(mileage_path, self.args)
         return self
 
     def add_consumption_data(self, data_name, df: pd.DataFrame) -> 'DataContainer':

simba/schedule.py

@@ -818,6 +818,7 @@ def rotation_filter(self, args, rf_list=[]):
                 warnings.warn(f"Path to rotation filter {args.rotation_filter} is invalid.")
                 # no file, no change
                 return
+        util.save_input_file(args.rotation_filter, args)
         # filter out rotations in self.rotations
         if args.rotation_filter_variable == "exclude":
             self.rotations = {k: v for k, v in self.rotations.items() if k not in rf_list}
@@ -869,6 +870,7 @@ def generate_scenario(self, args):
             if time_windows_path.exists():
                 with time_windows_path.open('r', encoding='utf-8') as f:
                     time_windows = util.uncomment_json_file(f)
+                    util.save_input_file(args.time_windows, args)
                 # convert time window strings to date/times
                 for grid_operator, grid_operator_seasons in time_windows.items():
                     for season, info in grid_operator_seasons.items():
@@ -995,6 +997,7 @@ def generate_scenario(self, args):
                         if local_generation:
                             local_generation = update_csv_file_info(local_generation, gc_name)
                             events["local_generation"][gc_name + " feed-in"] = local_generation
+                            util.save_input_file(local_generation["csv_file"], args)
                             # add PV component
                             photovoltaics[gc_name] = {
                                 "parent": gc_name,
@@ -1006,6 +1009,7 @@ def generate_scenario(self, args):
                         if fixed_load:
                             fixed_load = update_csv_file_info(fixed_load, gc_name)
                             events["fixed_load"][gc_name + " ext. load"] = fixed_load
+                            util.save_input_file(fixed_load["csv_file"], args)
 
                         # temporary lowering of grid connector max power during peak load windows
                         if time_windows is not None:
@@ -1086,6 +1090,7 @@ def generate_scenario(self, args):
             else:
                 # read prices from CSV, convert to events
                 prices = get_price_list_from_csv(price_csv)
+                util.save_input_file(price_csv["csv_file"], args)
                 events["grid_operator_signals"] += generate_event_list_from_prices(
                     prices, gc_name, start_simulation, stop_simulation,
                     price_csv.get('start_time'), price_csv.get('step_duration_s'))

simba/simulate.py

@@ -2,7 +2,7 @@
 import traceback
 from copy import deepcopy
 
-from simba import report, optimization, optimizer_util
+from simba import report, optimization, optimizer_util, util
 from simba.data_container import DataContainer
 from simba.costs import calculate_costs
 from simba.optimizer_util import read_config as read_optimizer_config
@@ -44,6 +44,7 @@ def pre_simulation(args, data_container: DataContainer):
     :return: schedule, args
     :rtype: simba.schedule.Schedule, Namespace
     """
+
     # Deepcopy args so original args do not get mutated
     args = deepcopy(args)
 
@@ -202,6 +203,7 @@ def _station_optimization(schedule, scenario, args, i, single_step: bool):
             conf = optimizer_util.OptimizerConfig().set_defaults()
         else:
             conf = read_optimizer_config(args.optimizer_config)
+            util.save_input_file(args.optimizer_config, args)
         if single_step:
             conf.early_return = True
         # Work on copies of the original schedule and scenario. In case of an exception the outer
@@ -223,7 +225,7 @@ def station_optimization(schedule, scenario, args, i):
     @staticmethod
     def station_optimization_single_step(schedule, scenario, args, i):
         """ Electrify only the station with the highest potential
-        
+
         :param schedule: Schedule
         :type schedule: simba.schedule.Schedule
         :param scenario: Scenario
@@ -232,7 +234,7 @@ def station_optimization_single_step(schedule, scenario, args, i):
         :type args: argparse.Namespace
         :param i: counter of modes for directory creation
         :return: schedule, scenario
-        
+
         """  # noqa
         return Mode._station_optimization(schedule, scenario, args, i, single_step=True)
 

simba/util.py

@@ -2,6 +2,8 @@
 import csv
 import json
 import logging
+from pathlib import Path
+import shutil
 import subprocess
 from datetime import datetime, timedelta
 
@@ -18,6 +20,34 @@ def save_version(file_path):
         f.write("Git Hash SimBA:" + get_git_revision_hash())
 
 
+def save_input_file(file_path, args):
+    """ Copy given file to output folder, to ensure reproducibility.
+
+    *file_path* must exist and *output_directory_input* must be set in *args*.
+    If either condition is not met or the file has already been copied, nothing is done.
+
+    :param file_path: source file
+    :type file_path: string or Path
+    :param args: general info, output_directory_input is required
+    :type args: Namespace
+    """
+    if file_path is None:
+        return
+    output_directory_input = vars(args).get("output_directory_input", None)
+    if output_directory_input is None:
+        # input directory was not created
+        return
+    source_path = Path(file_path)
+    target_path = output_directory_input / source_path.name
+    if not source_path.exists():
+        # original file missing
+        return
+    if target_path.exists():
+        # already saved
+        return
+    shutil.copy(source_path, target_path)
+
+
 def uncomment_json_file(f, char='//'):
     """ Remove comments from JSON file.
 
@@ -443,6 +473,8 @@ def get_parser():
     parser.add_argument('--rotation-filter-variable', default=None,
                         choices=[None, 'include', 'exclude'],
                         help='set mode for filtering schedule rotations')
+    parser.add_argument('--zip-output', '-z', action='store_true',
+                        help='compress output folder after simulation')
 
     # #### Charging strategy #####
     parser.add_argument('--preferred-charging-type', '-pct', default='depb',

tests/test_example.py

@@ -26,3 +26,28 @@ def test_example_cfg(self, tmp_path):
         assert subprocess.call([
             "python", "-m", "simba", "--config", dst
         ]) == 0
+
+        # make sure all required files have been copied to output folder
+        expected = [
+            'simba.cfg',
+            'program_version.txt',
+            'trips_example.csv',
+            'electrified_stations.json',
+            'vehicle_types.json',
+
+            'cost_params.json',
+            'default_level_of_loading_over_day.csv',
+            'default_temp_winter.csv',
+            'energy_consumption_example.csv',
+            'example_pv_feedin.csv',
+            'example_external_load.csv',
+            'price_timeseries.csv',
+            'time_windows.json',
+        ]
+        input_dir = next(tmp_path.glob('*/input_data'))
+        missing = list()
+        for file_name in expected:
+            if not (input_dir / file_name).exists():
+                missing.append(file_name)
+        if missing:
+            raise Exception("Missing input files in output directory: " + ', '.join(missing))

tests/test_schedule.py

@@ -250,7 +250,6 @@ def test_assign_vehicles_adaptive(self):
 
     def test_calculate_consumption(self, default_schedule_arguments):
         """ Test if calling the consumption calculation works
-
         :param default_schedule_arguments: basic arguments the schedule needs for creation
         """
         sys.argv = ["foo", "--config", str(example_root / "simba.cfg")]

tests/test_station_optimization.py

@@ -1,6 +1,6 @@
-import logging
 from copy import copy, deepcopy
 import json
+import logging
 from pathlib import Path
 import pytest
 import random
@@ -106,12 +106,12 @@ def setup_test(self, tmp_path):
         dst.write_text(src_text)
 
     def generate_datacontainer_args(self, trips_file_name="trips.csv"):
-        """ Check if running a basic example works and if a scenario object is returned.
+        """ Check if running a basic example works and return data container.
 
         :param trips_file_name: file name of the trips file. Has to be inside the test_input_file
             folder
         :type trips_file_name: str
-        :return: schedule, scenario, args"""
+        :return: data_container, args"""
 
         sys.argv = ["foo", "--config", str(example_root / "simba.cfg")]
         args = util.get_args()