From 600ab0a1b83a0fc1d002ec64120764170ba9a08b Mon Sep 17 00:00:00 2001 From: Sohambasu07 Date: Tue, 7 May 2024 12:32:50 +0200 Subject: [PATCH 1/5] jahs_bench task=task instead of task_id and fidelity min --- src/mfpbench/jahs/benchmark.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mfpbench/jahs/benchmark.py b/src/mfpbench/jahs/benchmark.py index 08b05d4..4ec4216 100644 --- a/src/mfpbench/jahs/benchmark.py +++ b/src/mfpbench/jahs/benchmark.py @@ -77,7 +77,7 @@ class JAHSResult(Result[JAHSConfig, int]): class JAHSBenchmark(Benchmark[JAHSConfig, JAHSResult, int], ABC): JAHS_FIDELITY_NAME: ClassVar[str] = "epoch" - JAHS_FIDELITY_RANGE: ClassVar[tuple[int, int, int]] = (3, 200, 1) + JAHS_FIDELITY_RANGE: ClassVar[tuple[int, int, int]] = (1, 200, 1) JAHS_METRICS_TO_ACTIVATE: ClassVar[tuple[str, ...]] = ( "valid-acc", "test-acc", @@ -198,7 +198,7 @@ def bench(self) -> jahs_bench.Benchmark: ) self._bench = jahs_bench.Benchmark( - task=self.task_id, + task=task, save_dir=self.datadir, download=False, metrics=self.JAHS_METRICS_TO_ACTIVATE, From 9ad821a42000946c9b6a321c63d0c3fd6319d0a6 Mon Sep 17 00:00:00 2001 From: Sohambasu07 Date: Tue, 7 May 2024 12:33:06 +0200 Subject: [PATCH 2/5] clipped surrogate results at max bounds --- src/mfpbench/pd1/benchmark.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/mfpbench/pd1/benchmark.py b/src/mfpbench/pd1/benchmark.py index d55dd75..9c497d4 100644 --- a/src/mfpbench/pd1/benchmark.py +++ b/src/mfpbench/pd1/benchmark.py @@ -201,7 +201,8 @@ def _results_for( features = xs.columns for metric, surrogate in self.surrogates.items(): # We clip as sometimes the surrogate produces negative values - xs[metric] = surrogate.predict(xs[features]).clip(min=0) + bounds = self.Result.metric_defs[metric].bounds + xs[metric] = surrogate.predict(xs[features]).clip(min=0, max=(bounds[1])) metrics = list(self.surrogates.keys()) return [dict(r[metrics]) for _, r in xs.iterrows()] From 592d5c0667aa90d8fdb323b5d9d7f7cf851ba059 Mon Sep 17 00:00:00 2001 From: Sohambasu07 Date: Tue, 7 May 2024 12:34:00 +0200 Subject: [PATCH 3/5] Corrected surrogate names and changed min fids to 1 --- src/mfpbench/pd1/benchmarks/cifar100.py | 4 ++-- src/mfpbench/pd1/benchmarks/imagenet.py | 2 +- src/mfpbench/pd1/benchmarks/translate_wmt.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/mfpbench/pd1/benchmarks/cifar100.py b/src/mfpbench/pd1/benchmarks/cifar100.py index 977772f..70b8860 100644 --- a/src/mfpbench/pd1/benchmarks/cifar100.py +++ b/src/mfpbench/pd1/benchmarks/cifar100.py @@ -6,9 +6,9 @@ class PD1cifar100_wideresnet_2048(PD1Benchmark): - pd1_fidelity_range = (45, 199, 1) + pd1_fidelity_range = (1, 199, 1) pd1_result_type = PD1ResultTransformer - pd1_name = "cifar100-wideresnet-2048" + pd1_name = "cifar100-wide_resnet-2048" @classmethod def _create_space(cls, seed: int | None = None) -> ConfigurationSpace: diff --git a/src/mfpbench/pd1/benchmarks/imagenet.py b/src/mfpbench/pd1/benchmarks/imagenet.py index cf1e818..4fbd3a7 100644 --- a/src/mfpbench/pd1/benchmarks/imagenet.py +++ b/src/mfpbench/pd1/benchmarks/imagenet.py @@ -7,7 +7,7 @@ class PD1imagenet_resnet_512(PD1Benchmark): pd1_result_type = PD1ResultTransformer - pd1_fidelity_range = (3, 99, 1) + pd1_fidelity_range = (1, 99, 1) pd1_name = "imagenet-resnet-512" @classmethod diff --git a/src/mfpbench/pd1/benchmarks/translate_wmt.py b/src/mfpbench/pd1/benchmarks/translate_wmt.py index dd14bf8..edd8729 100644 --- a/src/mfpbench/pd1/benchmarks/translate_wmt.py +++ b/src/mfpbench/pd1/benchmarks/translate_wmt.py @@ -8,7 +8,7 @@ class PD1translatewmt_xformer_64(PD1Benchmark): pd1_fidelity_range = (1, 19, 1) pd1_result_type = PD1ResultTransformer - pd1_name = "translate-wmt-xformer-64" + pd1_name = "translate_wmt-xformer_translate-64" @classmethod def _create_space(cls, seed: int | None = None) -> ConfigurationSpace: From 4e5dec656ed5c29769a20745182a9d13230328e3 Mon Sep 17 00:00:00 2001 From: Sohambasu07 Date: Tue, 7 May 2024 12:34:23 +0200 Subject: [PATCH 4/5] Changed min fid to 1 --- src/mfpbench/synthetic/hartmann/benchmark.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mfpbench/synthetic/hartmann/benchmark.py b/src/mfpbench/synthetic/hartmann/benchmark.py index 3362feb..107d12c 100644 --- a/src/mfpbench/synthetic/hartmann/benchmark.py +++ b/src/mfpbench/synthetic/hartmann/benchmark.py @@ -160,7 +160,7 @@ def __init__( config_type=self.mfh_config_type, result_type=self.mfh_result_type, fidelity_name="z", - fidelity_range=(3, _max_fidelity, 1), + fidelity_range=(1, _max_fidelity, 1), space=space, seed=seed, prior=prior, From 0cf25063ba35feebe0e4eb9aa0b0ee86688b6d27 Mon Sep 17 00:00:00 2001 From: Sohambasu07 Date: Tue, 7 May 2024 14:11:25 +0200 Subject: [PATCH 5/5] clip(*bounds) --- src/mfpbench/pd1/benchmark.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mfpbench/pd1/benchmark.py b/src/mfpbench/pd1/benchmark.py index 9c497d4..3989b73 100644 --- a/src/mfpbench/pd1/benchmark.py +++ b/src/mfpbench/pd1/benchmark.py @@ -202,7 +202,7 @@ def _results_for( for metric, surrogate in self.surrogates.items(): # We clip as sometimes the surrogate produces negative values bounds = self.Result.metric_defs[metric].bounds - xs[metric] = surrogate.predict(xs[features]).clip(min=0, max=(bounds[1])) + xs[metric] = surrogate.predict(xs[features]).clip(*bounds) metrics = list(self.surrogates.keys()) return [dict(r[metrics]) for _, r in xs.iterrows()]