diff --git a/simba/__main__.py b/simba/__main__.py index 619a2de..094d648 100644 --- a/simba/__main__.py +++ b/simba/__main__.py @@ -26,18 +26,9 @@ except NotADirectoryError: # can't create new directory (may be write protected): no output args.output_directory = None - if args.output_directory is not None: - # copy input files to output to ensure reproducibility - copy_list = [args.config, args.electrified_stations_path, args.vehicle_types_path] - if "station_optimization" in args.mode: - copy_list.append(args.optimizer_config) - - # only copy cost params if they exist - if args.cost_parameters_path is not None: - copy_list.append(args.cost_parameters_path) - for c_file in map(Path, copy_list): - shutil.copy(c_file, args.output_directory_input / c_file.name) + # copy basic input to output to ensure reproducibility + util.save_input_file(args.config, args) util.save_version(args.output_directory_input / "program_version.txt") util.setup_logging(args, time_str) @@ -49,3 +40,9 @@ raise finally: logging.shutdown() + if args.zip_output and args.output_directory is not None and args.output_directory.exists(): + # compress output directory after simulation + # generate .zip at location of original output directory + shutil.make_archive(args.output_directory, 'zip', args.output_directory) + # remove original output directory + shutil.rmtree(args.output_directory) diff --git a/simba/data_container.py b/simba/data_container.py index f41fd1d..22a6fcc 100644 --- a/simba/data_container.py +++ b/simba/data_container.py @@ -31,6 +31,8 @@ def __init__(self): # List of trip dictionaries containing trip information like arrival time and station # departure time and station, distance and more self.trip_data: [dict] = [] + # Simulation arguments + self.args = None def fill_with_args(self, args: argparse.Namespace) -> 'DataContainer': """ Fill DataContainer with data from file_paths defined in args. @@ -40,6 +42,7 @@ def fill_with_args(self, args: argparse.Namespace) -> 'DataContainer': level_of_loading_over_day_path, station_data_path :return: self """ + self.args = args return self.fill_with_paths( trips_file_path=args.input_schedule, @@ -69,6 +72,7 @@ def add_trip_data_from_csv(self, file_path: Path) -> 'DataContainer': trip_d[TEMPERATURE] = util.cast_float_or_none(trip.get(TEMPERATURE)) trip_d["distance"] = float(trip["distance"]) self.trip_data.append(trip_d) + util.save_input_file(file_path, self.args) return self def add_station_geo_data(self, data: dict) -> None: @@ -162,6 +166,7 @@ def add_station_geo_data_from_csv(self, file_path: Path) -> 'DataContainer': msg=f"Can't parse numeric data in line {line_num + 2} from file {file_path}.", level=100) raise + util.save_input_file(file_path, self.args) return self def add_level_of_loading_data(self, data: dict) -> 'DataContainer': @@ -186,6 +191,7 @@ def add_level_of_loading_data_from_csv(self, file_path: Path) -> 'DataContainer' index = "hour" column = "level_of_loading" level_of_loading_data_dict = util.get_dict_from_csv(column, file_path, index) + util.save_input_file(file_path, self.args) self.add_level_of_loading_data(level_of_loading_data_dict) return self @@ -209,6 +215,7 @@ def add_temperature_data_from_csv(self, file_path: Path) -> 'DataContainer': index = "hour" column = "temperature" temperature_data_dict = util.get_dict_from_csv(column, file_path, index) + util.save_input_file(file_path, self.args) self.add_temperature_data(temperature_data_dict) return self @@ -231,6 +238,7 @@ def add_cost_parameters_from_json(self, file_path: Path) -> 'DataContainer': :return: DataContainer containing cost parameters """ cost_parameters = self.get_json_from_file(file_path, "cost parameters") + util.save_input_file(file_path, self.args) self.add_cost_parameters(cost_parameters) return self @@ -253,6 +261,7 @@ def add_stations_from_json(self, file_path: Path) -> 'DataContainer': """ stations = self.get_json_from_file(file_path, "electrified stations") + util.save_input_file(file_path, self.args) self.add_stations(stations) return self @@ -274,6 +283,7 @@ def add_vehicle_types_from_json(self, file_path: Path): :return: DataContainer containing vehicle types """ vehicle_types = self.get_json_from_file(file_path, "vehicle types") + util.save_input_file(file_path, self.args) self.add_vehicle_types(vehicle_types) return self @@ -311,6 +321,7 @@ def add_consumption_data_from_vehicle_type_linked_files(self): delim = util.get_csv_delim(mileage_path) df = pd.read_csv(mileage_path, sep=delim) self.add_consumption_data(mileage_path, df) + util.save_input_file(mileage_path, self.args) return self def add_consumption_data(self, data_name, df: pd.DataFrame) -> 'DataContainer': diff --git a/simba/schedule.py b/simba/schedule.py index 341e3ba..77e55fe 100644 --- a/simba/schedule.py +++ b/simba/schedule.py @@ -816,6 +816,7 @@ def rotation_filter(self, args, rf_list=[]): warnings.warn(f"Path to rotation filter {args.rotation_filter} is invalid.") # no file, no change return + util.save_input_file(args.rotation_filter, args) # filter out rotations in self.rotations if args.rotation_filter_variable == "exclude": self.rotations = {k: v for k, v in self.rotations.items() if k not in rf_list} @@ -867,6 +868,7 @@ def generate_scenario(self, args): if time_windows_path.exists(): with time_windows_path.open('r', encoding='utf-8') as f: time_windows = util.uncomment_json_file(f) + util.save_input_file(args.time_windows, args) # convert time window strings to date/times for grid_operator, grid_operator_seasons in time_windows.items(): for season, info in grid_operator_seasons.items(): @@ -993,6 +995,7 @@ def generate_scenario(self, args): if local_generation: local_generation = update_csv_file_info(local_generation, gc_name) events["local_generation"][gc_name + " feed-in"] = local_generation + util.save_input_file(local_generation["csv_file"], args) # add PV component photovoltaics[gc_name] = { "parent": gc_name, @@ -1004,6 +1007,7 @@ def generate_scenario(self, args): if fixed_load: fixed_load = update_csv_file_info(fixed_load, gc_name) events["fixed_load"][gc_name + " ext. load"] = fixed_load + util.save_input_file(fixed_load["csv_file"], args) # temporary lowering of grid connector max power during peak load windows if time_windows is not None: @@ -1084,6 +1088,7 @@ def generate_scenario(self, args): else: # read prices from CSV, convert to events prices = get_price_list_from_csv(price_csv) + util.save_input_file(price_csv["csv_file"], args) events["grid_operator_signals"] += generate_event_list_from_prices( prices, gc_name, start_simulation, stop_simulation, price_csv.get('start_time'), price_csv.get('step_duration_s')) diff --git a/simba/simulate.py b/simba/simulate.py index 01139e8..b831cb6 100644 --- a/simba/simulate.py +++ b/simba/simulate.py @@ -2,7 +2,7 @@ import traceback from copy import deepcopy -from simba import report, optimization, optimizer_util +from simba import report, optimization, optimizer_util, util from simba.data_container import DataContainer from simba.costs import calculate_costs from simba.optimizer_util import read_config as read_optimizer_config @@ -44,6 +44,7 @@ def pre_simulation(args, data_container: DataContainer): :return: schedule, args :rtype: simba.schedule.Schedule, Namespace """ + # Deepcopy args so original args do not get mutated args = deepcopy(args) @@ -202,6 +203,7 @@ def _station_optimization(schedule, scenario, args, i, single_step: bool): conf = optimizer_util.OptimizerConfig().set_defaults() else: conf = read_optimizer_config(args.optimizer_config) + util.save_input_file(args.optimizer_config, args) if single_step: conf.early_return = True # Work on copies of the original schedule and scenario. In case of an exception the outer @@ -223,7 +225,7 @@ def station_optimization(schedule, scenario, args, i): @staticmethod def station_optimization_single_step(schedule, scenario, args, i): """ Electrify only the station with the highest potential - + :param schedule: Schedule :type schedule: simba.schedule.Schedule :param scenario: Scenario @@ -232,7 +234,7 @@ def station_optimization_single_step(schedule, scenario, args, i): :type args: argparse.Namespace :param i: counter of modes for directory creation :return: schedule, scenario - + """ # noqa return Mode._station_optimization(schedule, scenario, args, i, single_step=True) diff --git a/simba/util.py b/simba/util.py index 93feacb..b999c3c 100644 --- a/simba/util.py +++ b/simba/util.py @@ -2,6 +2,8 @@ import csv import json import logging +from pathlib import Path +import shutil import subprocess from datetime import datetime, timedelta @@ -18,6 +20,34 @@ def save_version(file_path): f.write("Git Hash SimBA:" + get_git_revision_hash()) +def save_input_file(file_path, args): + """ Copy given file to output folder, to ensure reproducibility. + + *file_path* must exist and *output_directory_input* must be set in *args*. + If either condition is not met or the file has already been copied, nothing is done. + + :param file_path: source file + :type file_path: string or Path + :param args: general info, output_directory_input is required + :type args: Namespace + """ + if file_path is None: + return + output_directory_input = vars(args).get("output_directory_input", None) + if output_directory_input is None: + # input directory was not created + return + source_path = Path(file_path) + target_path = output_directory_input / source_path.name + if not source_path.exists(): + # original file missing + return + if target_path.exists(): + # already saved + return + shutil.copy(source_path, target_path) + + def uncomment_json_file(f, char='//'): """ Remove comments from JSON file. @@ -428,6 +458,8 @@ def get_parser(): parser.add_argument('--rotation-filter-variable', default=None, choices=[None, 'include', 'exclude'], help='set mode for filtering schedule rotations') + parser.add_argument('--zip-output', '-z', action='store_true', + help='compress output folder after simulation') # #### Charging strategy ##### parser.add_argument('--preferred-charging-type', '-pct', default='depb', diff --git a/tests/test_example.py b/tests/test_example.py index 170f634..a2ee9f7 100644 --- a/tests/test_example.py +++ b/tests/test_example.py @@ -26,3 +26,28 @@ def test_example_cfg(self, tmp_path): assert subprocess.call([ "python", "-m", "simba", "--config", dst ]) == 0 + + # make sure all required files have been copied to output folder + expected = [ + 'simba.cfg', + 'program_version.txt', + 'trips_example.csv', + 'electrified_stations.json', + 'vehicle_types.json', + + 'cost_params.json', + 'default_level_of_loading_over_day.csv', + 'default_temp_winter.csv', + 'energy_consumption_example.csv', + 'example_pv_feedin.csv', + 'example_external_load.csv', + 'price_timeseries.csv', + 'time_windows.json', + ] + input_dir = next(tmp_path.glob('*/input_data')) + missing = list() + for file_name in expected: + if not (input_dir / file_name).exists(): + missing.append(file_name) + if missing: + raise Exception("Missing input files in output directory: " + ', '.join(missing)) diff --git a/tests/test_schedule.py b/tests/test_schedule.py index 57a5730..56385f2 100644 --- a/tests/test_schedule.py +++ b/tests/test_schedule.py @@ -250,7 +250,6 @@ def test_assign_vehicles_adaptive(self): def test_calculate_consumption(self, default_schedule_arguments): """ Test if calling the consumption calculation works - :param default_schedule_arguments: basic arguments the schedule needs for creation """ sys.argv = ["foo", "--config", str(example_root / "simba.cfg")] diff --git a/tests/test_station_optimization.py b/tests/test_station_optimization.py index 3fa822d..fe2f859 100644 --- a/tests/test_station_optimization.py +++ b/tests/test_station_optimization.py @@ -1,6 +1,6 @@ -import logging from copy import copy, deepcopy import json +import logging from pathlib import Path import pytest import random @@ -106,12 +106,12 @@ def setup_test(self, tmp_path): dst.write_text(src_text) def generate_datacontainer_args(self, trips_file_name="trips.csv"): - """ Check if running a basic example works and if a scenario object is returned. + """ Check if running a basic example works and return data container. :param trips_file_name: file name of the trips file. Has to be inside the test_input_file folder :type trips_file_name: str - :return: schedule, scenario, args""" + :return: data_container, args""" sys.argv = ["foo", "--config", str(example_root / "simba.cfg")] args = util.get_args()