From f2582b2cc9289e2d733f4b9d50cf180154720f59 Mon Sep 17 00:00:00 2001 From: Thomas Weise Date: Fri, 15 Nov 2024 19:28:03 +0800 Subject: [PATCH] move to new pycommons and moptipy with generator-based csv api --- moptipyapps/binpacking2d/packing_result.py | 122 +++++++++--------- .../binpacking2d/packing_statistics.py | 121 +++++++++-------- moptipyapps/shared.py | 24 ++-- moptipyapps/version.py | 2 +- pyproject.toml | 2 +- requirements-dev.txt | 2 +- requirements.txt | 4 +- setup.cfg | 4 +- .../test_binpacking2d_experiment.py | 12 +- ...packing2d_packing_result_and_statistics.py | 4 +- 10 files changed, 142 insertions(+), 155 deletions(-) diff --git a/moptipyapps/binpacking2d/packing_result.py b/moptipyapps/binpacking2d/packing_result.py index 92c736a9..057d2809 100644 --- a/moptipyapps/binpacking2d/packing_result.py +++ b/moptipyapps/binpacking2d/packing_result.py @@ -400,35 +400,32 @@ def to_csv(results: Iterable[PackingResult], file: str) -> Path: logger(f"Writing packing results to CSV file {path!r}.") path.ensure_parent_dir_exists() with path.open_for_write() as wt: - csv_write( - data=sorted(results), consumer=line_writer(wt), - setup=CsvWriter().setup, - get_column_titles=CsvWriter.get_column_titles, - get_row=CsvWriter.get_row, - get_header_comments=CsvWriter.get_header_comments, - get_footer_comments=CsvWriter.get_footer_comments, - get_footer_bottom_comments=CsvWriter.get_footer_bottom_comments) + consumer: Final[Callable[[str], None]] = line_writer(wt) + for p in csv_write( + data=sorted(results), + setup=CsvWriter().setup, + column_titles=CsvWriter.get_column_titles, + get_row=CsvWriter.get_row, + header_comments=CsvWriter.get_header_comments, + footer_comments=CsvWriter.get_footer_comments, + footer_bottom_comments=CsvWriter.get_footer_bottom_comments): + consumer(p) logger(f"Done writing packing results to CSV file {path!r}.") return path -def from_csv(file: str, - consumer: Callable[[PackingResult], None]) -> None: +def from_csv(file: str) -> Iterable[PackingResult]: """ Load the packing results from a CSV file. :param file: the file to read from - :param consumer: the consumer for the results + :returns: the iterable with the packing result """ - if not callable(consumer): - raise type_error(consumer, "consumer", call=True) path: Final[Path] = file_path(file) logger(f"Now reading CSV file {path!r}.") with path.open_for_read() as rd: - csv_read(rows=rd, - setup=CsvReader, - parse_row=CsvReader.parse_row, - consumer=consumer) + yield from csv_read( + rows=rd, setup=CsvReader, parse_row=CsvReader.parse_row) logger(f"Done reading CSV file {path!r}.") @@ -480,99 +477,98 @@ def setup(self, data: Iterable[PackingResult]) -> "CsvWriter": return self - def get_column_titles(self, dest: Callable[[str], None]) -> None: + def get_column_titles(self) -> Iterable[str]: """ Get the column titles. - :param dest: the destination string consumer + :returns: the column titles """ p: Final[str] = self.scope - self.__er.get_column_titles(dest) + yield from self.__er.get_column_titles() - dest(csv_scope(p, KEY_BIN_HEIGHT)) - dest(csv_scope(p, KEY_BIN_WIDTH)) - dest(csv_scope(p, KEY_N_ITEMS)) - dest(csv_scope(p, KEY_N_DIFFERENT_ITEMS)) + yield csv_scope(p, KEY_BIN_HEIGHT) + yield csv_scope(p, KEY_BIN_WIDTH) + yield csv_scope(p, KEY_N_ITEMS) + yield csv_scope(p, KEY_N_DIFFERENT_ITEMS) if self.__bin_bounds: for b in self.__bin_bounds: - dest(csv_scope(p, b)) + yield csv_scope(p, b) if self.__objectives: for o in self.__objectives: oo: str = csv_scope(p, o) - dest(csv_scope(oo, _OBJECTIVE_LOWER)) - dest(oo) - dest(csv_scope(oo, _OBJECTIVE_UPPER)) + yield csv_scope(oo, _OBJECTIVE_LOWER) + yield oo + yield csv_scope(oo, _OBJECTIVE_UPPER) - def get_row(self, data: PackingResult, - dest: Callable[[str], None]) -> None: + def get_row(self, data: PackingResult) -> Iterable[str]: """ Render a single packing result record to a CSV row. :param data: the end result record - :param dest: the string consumer + :returns: the iterable with the row data """ - self.__er.get_row(data.end_result, dest) - dest(repr(data.bin_height)) - dest(repr(data.bin_width)) - dest(repr(data.n_items)) - dest(repr(data.n_different_items)) + yield from self.__er.get_row(data.end_result) + yield repr(data.bin_height) + yield repr(data.bin_width) + yield repr(data.n_items) + yield repr(data.n_different_items) if self.__bin_bounds: for bb in self.__bin_bounds: - dest(repr(data.bin_bounds[bb]) - if bb in data.bin_bounds else "") + yield (repr(data.bin_bounds[bb]) + if bb in data.bin_bounds else "") if self.__objectives: for ob in self.__objectives: ox = csv_scope(ob, _OBJECTIVE_LOWER) - dest(num_to_str(data.objective_bounds[ox]) - if ox in data.objective_bounds else "") - dest(num_to_str(data.objectives[ob]) - if ob in data.objectives else "") + yield (num_to_str(data.objective_bounds[ox]) + if ox in data.objective_bounds else "") + yield (num_to_str(data.objectives[ob]) + if ob in data.objectives else "") ox = csv_scope(ob, _OBJECTIVE_UPPER) - dest(num_to_str(data.objective_bounds[ox]) - if ox in data.objective_bounds else "") + yield (num_to_str(data.objective_bounds[ox]) + if ox in data.objective_bounds else "") - def get_header_comments(self, dest: Callable[[str], None]) -> None: + def get_header_comments(self) -> Iterable[str]: """ Get any possible header comments. - :param dest: the destination + :returns: the header comments """ - dest("End Results of Bin Packing Experiments") - dest("See the description at the bottom of the file.") + return ("End Results of Bin Packing Experiments", + "See the description at the bottom of the file.") - def get_footer_comments(self, dest: Callable[[str], None]) -> None: + def get_footer_comments(self) -> Iterable[str]: """ Get any possible footer comments. - :param dest: the destination + :return: the footer comments """ - self.__er.get_footer_comments(dest) - dest("") + yield from self.__er.get_footer_comments() + yield "" p: Final[str | None] = self.scope if self.__bin_bounds: for bb in self.__bin_bounds: - dest(f"{csv_scope(p, bb)} is a lower bound " - "for the number of bins.") + yield (f"{csv_scope(p, bb)} is a lower bound " + f"for the number of bins.") if self.__objectives: for obb in self.__objectives: ob: str = csv_scope(p, obb) ox: str = csv_scope(ob, _OBJECTIVE_LOWER) - dest(f"{ox}: a lower bound of the {ob} objective function.") - dest(f"{ob}: one of the possible objective functions for the " - "two-dimensional bin packing problem.") + yield f"{ox}: a lower bound of the {ob} objective function." + yield (f"{ob}: one of the possible objective functions for " + "the two-dimensional bin packing problem.") ox = csv_scope(ob, _OBJECTIVE_UPPER) - dest(f"{ox}: an upper bound of the {ob} objective function.") + yield f"{ox}: an upper bound of the {ob} objective function." - def get_footer_bottom_comments(self, dest: Callable[[str], None]) -> None: + def get_footer_bottom_comments(self) -> Iterable[str]: """ Get the bottom footer comments. :param dest: the destination """ - motipyapps_footer_bottom_comments( - self, dest, "The packing data is assembled using module " - "moptipyapps.binpacking2d.packing_statistics.") - ErCsvWriter.get_footer_bottom_comments(self.__er, dest) + yield from motipyapps_footer_bottom_comments( + self, "The packing data is assembled using module " + "moptipyapps.binpacking2d.packing_statistics.") + yield from ErCsvWriter.get_footer_bottom_comments(self.__er) class CsvReader: diff --git a/moptipyapps/binpacking2d/packing_statistics.py b/moptipyapps/binpacking2d/packing_statistics.py index 416b3498..227c8fc7 100644 --- a/moptipyapps/binpacking2d/packing_statistics.py +++ b/moptipyapps/binpacking2d/packing_statistics.py @@ -312,35 +312,33 @@ def to_csv(results: Iterable[PackingStatistics], file: str) -> Path: logger(f"Writing packing statistics to CSV file {path!r}.") path.ensure_parent_dir_exists() with path.open_for_write() as wt: - csv_write( - data=sorted(results), consumer=line_writer(wt), - setup=CsvWriter().setup, - get_column_titles=CsvWriter.get_column_titles, - get_row=CsvWriter.get_row, - get_header_comments=CsvWriter.get_header_comments, - get_footer_comments=CsvWriter.get_footer_comments, - get_footer_bottom_comments=CsvWriter.get_footer_bottom_comments) + consumer: Final[Callable[[str], None]] = line_writer(wt) + for p in csv_write( + data=sorted(results), + setup=CsvWriter().setup, + column_titles=CsvWriter.get_column_titles, + get_row=CsvWriter.get_row, + header_comments=CsvWriter.get_header_comments, + footer_comments=CsvWriter.get_footer_comments, + footer_bottom_comments=CsvWriter.get_footer_bottom_comments): + consumer(p) logger(f"Done writing packing statistics to CSV file {path!r}.") return path -def from_csv(file: str, - consumer: Callable[[PackingStatistics], None]) -> None: +def from_csv(file: str) -> Iterable[PackingStatistics]: """ Load the packing statistics from a CSV file. :param file: the file to read from - :param consumer: the consumer for the statistics + :returns: the iterable with the packing statistics """ - if not callable(consumer): - raise type_error(consumer, "consumer", call=True) path: Final[Path] = file_path(file) logger(f"Now reading CSV file {path!r}.") with path.open_for_read() as rd: - csv_read(rows=rd, - setup=CsvReader, - parse_row=CsvReader.parse_row, - consumer=consumer) + yield from csv_read(rows=rd, + setup=CsvReader, + parse_row=CsvReader.parse_row) logger(f"Done reading CSV file {path!r}.") @@ -408,101 +406,100 @@ def setup(self, data: Iterable[PackingStatistics]) -> "CsvWriter": return self - def get_column_titles(self, dest: Callable[[str], None]) -> None: + def get_column_titles(self) -> Iterable[str]: """ Get the column titles. :param dest: the destination string consumer """ p: Final[str | None] = self.scope - self.__es.get_column_titles(dest) + yield from self.__es.get_column_titles() - dest(csv_scope(p, KEY_BIN_HEIGHT)) - dest(csv_scope(p, KEY_BIN_WIDTH)) - dest(csv_scope(p, KEY_N_ITEMS)) - dest(csv_scope(p, KEY_N_DIFFERENT_ITEMS)) + yield csv_scope(p, KEY_BIN_HEIGHT) + yield csv_scope(p, KEY_BIN_WIDTH) + yield csv_scope(p, KEY_N_ITEMS) + yield csv_scope(p, KEY_N_DIFFERENT_ITEMS) if self.__bin_bounds: for b in self.__bin_bounds: - dest(csv_scope(p, b)) + yield csv_scope(p, b) if self.__objective_names and self.__objectives: for i, o in enumerate(self.__objectives): - dest(csv_scope(p, self.__objective_lb_names[i])) - o.get_column_titles(dest) - dest(csv_scope(p, self.__objective_ub_names[i])) + yield csv_scope(p, self.__objective_lb_names[i]) + yield from o.get_column_titles() + yield csv_scope(p, self.__objective_ub_names[i]) - def get_row(self, data: PackingStatistics, - dest: Callable[[str], None]) -> None: + def get_row(self, data: PackingStatistics) -> Iterable[str]: """ Render a single packing result record to a CSV row. :param data: the end result record - :param dest: the string consumer + :returns: the iterable with the row text """ - self.__es.get_row(data.end_statistics, dest) - dest(repr(data.bin_height)) - dest(repr(data.bin_width)) - dest(repr(data.n_items)) - dest(repr(data.n_different_items)) + yield from self.__es.get_row(data.end_statistics) + yield repr(data.bin_height) + yield repr(data.bin_width) + yield repr(data.n_items) + yield repr(data.n_different_items) if self.__bin_bounds: for bb in self.__bin_bounds: - dest(repr(data.bin_bounds[bb]) - if bb in data.bin_bounds else "") + yield (repr(data.bin_bounds[bb]) + if bb in data.bin_bounds else "") if self.__objective_names and self.__objectives: lb: Final[tuple[str, ...] | None] = self.__objective_lb_names ub: Final[tuple[str, ...] | None] = self.__objective_ub_names for i, ob in enumerate(self.__objective_names): if lb is not None: ox = lb[i] - dest(num_to_str(data.objective_bounds[ox]) - if ox in data.objective_bounds else "") - SsCsvWriter.get_optional_row( - self.__objectives[i], data.objectives.get(ob), dest) + yield (num_to_str(data.objective_bounds[ox]) + if ox in data.objective_bounds else "") + yield from SsCsvWriter.get_optional_row( + self.__objectives[i], data.objectives.get(ob)) if ub is not None: ox = ub[i] - dest(num_to_str(data.objective_bounds[ox]) - if ox in data.objective_bounds else "") + yield (num_to_str(data.objective_bounds[ox]) + if ox in data.objective_bounds else "") - def get_header_comments(self, dest: Callable[[str], None]) -> None: + def get_header_comments(self) -> Iterable[str]: """ Get any possible header comments. - :param dest: the destination + :returns: the header comments """ - dest("End Statistics of Bin Packing Experiments") - dest("See the description at the bottom of the file.") + return ("End Statistics of Bin Packing Experiments", + "See the description at the bottom of the file.") - def get_footer_comments(self, dest: Callable[[str], None]) -> None: + def get_footer_comments(self) -> Iterable[str]: """ Get any possible footer comments. - :param dest: the destination + :returns: the footer comments """ - self.__es.get_footer_comments(dest) - dest("") + yield from self.__es.get_footer_comments() + yield "" p: Final[str | None] = self.scope if self.__bin_bounds: for bb in self.__bin_bounds: - dest(f"{csv_scope(p, bb)} is a lower " - "bound for the number of bins.") + yield (f"{csv_scope(p, bb)} is a lower " + "bound for the number of bins.") if self.__objectives and self.__objective_names: for i, obb in enumerate(self.__objective_names): ob: str = csv_scope(p, obb) ox: str = csv_scope(ob, _OBJECTIVE_LOWER) - dest(f"{ox}: a lower bound of the {ob} objective function.") - self.__objectives[i].get_footer_comments(dest) + yield f"{ox}: a lower bound of the {ob} objective function." + yield from self.__objectives[i].get_footer_comments() ox = csv_scope(ob, _OBJECTIVE_UPPER) - dest(f"{ox}: an upper bound of the {ob} objective function.") + yield f"{ox}: an upper bound of the {ob} objective function." - def get_footer_bottom_comments(self, dest: Callable[[str], None]) -> None: + def get_footer_bottom_comments(self) -> Iterable[str]: """ Get the bottom footer comments. :param dest: the destination """ - motipyapps_footer_bottom_comments( - self, dest, "The packing data is assembled using module " - "moptipyapps.binpacking2d.packing_statistics.") - EsCsvWriter.get_footer_bottom_comments(self.__es, dest) + yield from motipyapps_footer_bottom_comments( + self, "The packing data is assembled using module " + "moptipyapps.binpacking2d.packing_statistics.") + yield from EsCsvWriter.get_footer_bottom_comments(self.__es) class CsvReader: @@ -604,7 +601,7 @@ def parse_row(self, data: list[str]) -> PackingStatistics: packing_results: Final[list[PackingResult]] = [] if src_path.is_file(): logger(f"{src_path!r} identifies as file, load as end-results csv") - pr_from_csv(src_path, packing_results.append) + packing_results.extend(pr_from_csv(src_path)) else: logger(f"{src_path!r} identifies as directory, load it as log files") pr_from_logs(src_path, packing_results.append) diff --git a/moptipyapps/shared.py b/moptipyapps/shared.py index 0ecbfd1e..738f30c3 100644 --- a/moptipyapps/shared.py +++ b/moptipyapps/shared.py @@ -1,7 +1,7 @@ """Some shared variables and constants.""" import argparse -from typing import Any, Callable, Final +from typing import Any, Final, Iterable import moptipy.examples.jssp.instance as ins from pycommons.io.arguments import make_argparser, make_epilog @@ -38,30 +38,28 @@ def moptipyapps_argparser(file: str, description: str, def motipyapps_footer_bottom_comments( - _: Any, dest: Callable[[str], Any], - additional: str | None = None) -> None: + _: Any, additional: str | None = None) -> Iterable[str]: """ Print the standard csv footer for moptipyapps. :param _: the setup object, ignored - :param dest: the destination callable - :param dest: the destination to write to :param additional: any additional output string + :returns: the comments - >>> def __qpt(s: str): + >>> for s in motipyapps_footer_bottom_comments(None, "bla"): ... print(s[:49]) - >>> motipyapps_footer_bottom_comments(None, __qpt, "bla") This data has been generated with moptipyapps ver bla You can find moptipyapps at https://thomasweise.g - >>> motipyapps_footer_bottom_comments(None, __qpt, None) + >>> for s in motipyapps_footer_bottom_comments(None, None): + ... print(s[:49]) This data has been generated with moptipyapps ver You can find moptipyapps at https://thomasweise.g """ - dest("This data has been generated with moptipyapps version " - f"{moptipyapps_version}.") + yield ("This data has been generated with moptipyapps version " + f"{moptipyapps_version}.") if (additional is not None) and (str.__len__(additional) > 0): - dest(additional) - dest("You can find moptipyapps at " - "https://thomasweise.github.io/moptipyapps.") + yield additional + yield ("You can find moptipyapps at " + "https://thomasweise.github.io/moptipyapps.") diff --git a/moptipyapps/version.py b/moptipyapps/version.py index 22c3b784..7af52f77 100644 --- a/moptipyapps/version.py +++ b/moptipyapps/version.py @@ -1,4 +1,4 @@ """An internal file with the version of the `moptipyapps` package.""" from typing import Final -__version__: Final[str] = "0.8.61" +__version__: Final[str] = "0.8.62" diff --git a/pyproject.toml b/pyproject.toml index 87ad5347..38c2da37 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,3 @@ [build-system] -requires = ["setuptools>=71.1.0"] +requires = ["setuptools>=75.5.0"] build-backend = "setuptools.build_meta" diff --git a/requirements-dev.txt b/requirements-dev.txt index fac073d5..848c18ba 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -33,4 +33,4 @@ # # pycommons provides lots of utilities -pycommons[dev] >= 0.8.57 +pycommons[dev] >= 0.8.58 diff --git a/requirements.txt b/requirements.txt index 27f4286e..9cc9b1be 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,10 +22,10 @@ # `moptipy` provides the basic optimization infrastructure and the spaces and # tools that we use for optimization. -moptipy == 0.9.132 +moptipy == 0.9.136 # the common tools package -pycommons == 0.8.57 +pycommons == 0.8.58 # `numpy` is needed for its efficient data structures. numpy == 1.26.4 diff --git a/setup.cfg b/setup.cfg index 4e6b75d8..a95ddede 100644 --- a/setup.cfg +++ b/setup.cfg @@ -41,11 +41,11 @@ include_package_data = True install_requires = certifi >= 2024.8.30 defusedxml >= 0.7.1 - moptipy >= 0.9.132 + moptipy >= 0.9.136 numpy < 2 numba >= 0.60.0 matplotlib >= 3.9.2 - pycommons >= 0.8.57 + pycommons >= 0.8.58 scipy >= 1.14.1 urllib3 >= 2.2.3 packages = find: diff --git a/tests/binpacking2d/test_binpacking2d_experiment.py b/tests/binpacking2d/test_binpacking2d_experiment.py index a4338041..4521696b 100644 --- a/tests/binpacking2d/test_binpacking2d_experiment.py +++ b/tests/binpacking2d/test_binpacking2d_experiment.py @@ -137,8 +137,7 @@ def __evaluate(results: Path, evaluation: Path) -> None: assert N_RUNS <= len(seeds_1) <= N_RUNS * len(INSTANCES) p = Path.resolve_inside(evaluation, "results.txt") er_to_csv(end_results, p) - end_results_2: Final[list[EndResult]] = [] - er_from_csv(p, end_results_2.append) + end_results_2: Final[list[EndResult]] = list(er_from_csv(p)) assert sorted(end_results, key=lambda er: ( er.algorithm, er.instance, er.rand_seed)) == sorted( @@ -155,8 +154,7 @@ def __evaluate(results: Path, evaluation: Path) -> None: assert set(INSTANCES) == instances_2 p = Path.resolve_inside(evaluation, "statistics.txt") es_to_csv(end_statistics, p) - end_statistics_2: Final[list[EndStatistics]] = [] - es_from_csv(p, end_statistics_2.append) + end_statistics_2: Final[list[EndStatistics]] = list(es_from_csv(p)) assert sorted( end_statistics, key=lambda er: ( er.algorithm, er.instance)) == sorted( @@ -179,8 +177,7 @@ def __evaluate(results: Path, evaluation: Path) -> None: assert seeds_2 == seeds_1 p = Path.resolve_inside(evaluation, "pack_results.txt") pr_to_csv(packing_results, p) - packing_results_2: Final[list[PackingResult]] = [] - pr_from_csv(p, packing_results_2.append) + packing_results_2: Final[list[PackingResult]] = list(pr_from_csv(p)) assert sorted(packing_results, key=lambda er: ( er.end_result.algorithm, er.end_result.instance, er.end_result.rand_seed)) == sorted( @@ -200,8 +197,7 @@ def __evaluate(results: Path, evaluation: Path) -> None: assert algorithms_4 == algorithms_1 p = Path.resolve_inside(evaluation, "pack_statistics.txt") ps_to_csv(packing_statistics, p) - ps2: Final[list[PackingStatistics]] = [] - ps_from_csv(p, ps2.append) + ps2: Final[list[PackingStatistics]] = list(ps_from_csv(p)) assert ps2 == packing_statistics diff --git a/tests/binpacking2d/test_binpacking2d_packing_result_and_statistics.py b/tests/binpacking2d/test_binpacking2d_packing_result_and_statistics.py index 79d4de1c..763ffa00 100644 --- a/tests/binpacking2d/test_binpacking2d_packing_result_and_statistics.py +++ b/tests/binpacking2d/test_binpacking2d_packing_result_and_statistics.py @@ -119,7 +119,7 @@ def test_packing_results_experiment() -> None: assert len(all_objectives) == 2 with temp_file() as tf: pr_to_csv(results_1, tf) - pr_from_csv(tf, results_2.append) + results_2.extend(pr_from_csv(tf)) assert len(results_2) == len(results_1) results_2.sort() assert results_1 == results_2 @@ -133,5 +133,5 @@ def test_packing_results_experiment() -> None: end_stats_3: Final[list[PackingStatistics]] = [] with temp_file() as tf2: ps_to_csv(end_stats_1, tf2) - ps_from_csv(tf2, end_stats_3.append) + end_stats_3.extend(ps_from_csv(tf2)) assert end_stats_3 == end_stats_2