Adding in CPU time to wherever time is defined as variable "cpu_time" (…

…#1173)

* Adding in CPU time to wherever time is defined as variable "cpu_time"

* Fix changelog

* Updating comments as requested in PR

* Forgot to update comment

* Adding in CPU time tests

CHANGELOG.md

@@ -10,6 +10,7 @@
 ## Improvements
 - Add logger information on handling of stopIteration error (#960)
 - Replace deprecated ConfigSpace methods (#1139)
+- Separated Wallclock time measurements from CPU time measurements and storing them under new 'cpu_time' variable (#1173)
 
 ## Dependencies
 - Allow numpy >= 2.x (#1146)

smac/main/smbo.py

@@ -81,6 +81,7 @@ def __init__(
         # Stats variables
         self._start_time: float | None = None
         self._used_target_function_walltime = 0.0
+        self._used_target_function_cputime = 0.0
 
         # Set walltime used method for intensifier
         self._intensifier.used_walltime = lambda: self.used_walltime  # type: ignore
@@ -108,7 +109,7 @@ def remaining_walltime(self) -> float:
     @property
     def remaining_cputime(self) -> float:
         """Subtracts the target function running budget with the used time."""
-        return self._scenario.cputime_limit - self._used_target_function_walltime
+        return self._scenario.cputime_limit - self._used_target_function_cputime
 
     @property
     def remaining_trials(self) -> int:
@@ -137,6 +138,11 @@ def used_target_function_walltime(self) -> float:
         """Returns how much walltime the target function spend so far."""
         return self._used_target_function_walltime
 
+    @property
+    def used_target_function_cputime(self) -> float:
+        """Returns how much time the target function spend on the hardware so far."""
+        return self._used_target_function_cputime
+
     def ask(self) -> TrialInfo:
         """Asks the intensifier for the next trial.
 
@@ -204,6 +210,7 @@ def tell(
             config=info.config,
             cost=value.cost,
             time=value.time,
+            cpu_time=value.cpu_time,
             status=value.status,
             instance=info.instance,
             seed=info.seed,
@@ -355,6 +362,7 @@ def optimize(self, *, data_to_scatter: dict[str, Any] | None = None) -> Configur
     def reset(self) -> None:
         """Resets the internal variables of the optimizer, intensifier, and runhistory."""
         self._used_target_function_walltime = 0
+        self._used_target_function_cputime = 0
         self._finished = False
 
         # We also reset runhistory and intensifier here
@@ -398,6 +406,7 @@ def load(self) -> None:
             self._intensifier.load(intensifier_fn)
 
             self._used_target_function_walltime = data["used_target_function_walltime"]
+            self._used_target_function_cputime = data["used_target_function_cputime"]
             self._finished = data["finished"]
             self._start_time = time.time() - data["used_walltime"]
 
@@ -409,6 +418,7 @@ def save(self) -> None:
             data = {
                 "used_walltime": self.used_walltime,
                 "used_target_function_walltime": self.used_target_function_walltime,
+                "used_target_function_cputime": self.used_target_function_cputime,
                 "last_update": time.time(),
                 "finished": self._finished,
             }
@@ -442,6 +452,7 @@ def _add_results(self) -> None:
 
             # Update SMAC stats
             self._used_target_function_walltime += float(trial_value.time)
+            self._used_target_function_cputime += float(trial_value.cpu_time)
 
             # Gracefully end optimization if termination cost is reached
             if self._scenario.termination_cost_threshold != np.inf:
@@ -582,7 +593,7 @@ def validate(
 
             # TODO: Use submit run for faster evaluation
             # self._runner.submit_trial(trial_info=trial)
-            _, cost, _, _ = self._runner.run(config, **kwargs)
+            _, cost, _, _, _ = self._runner.run(config, **kwargs)
             costs += [cost]
 
         np_costs = np.array(costs)
@@ -600,5 +611,7 @@ def print_stats(self) -> None:
             f"--- Used wallclock time: {round(self.used_walltime)} / {self._scenario.walltime_limit} sec\n"
             "--- Used target function runtime: "
             f"{round(self.used_target_function_walltime, 2)} / {self._scenario.cputime_limit} sec\n"
+            "--- Used target function CPU time: "
+            f"{round(self.used_target_function_cputime, 2)} / {self._scenario.cputime_limit} sec\n"
             f"----------------------------------------------------"
         )

smac/runhistory/dataclasses.py

@@ -98,6 +98,8 @@ class TrialValue:
     ----------
     cost : float | list[float]
     time : float, defaults to 0.0
+    cpu_time : float, defaults to 0.0
+        Describes the amount of time the trial spend on hardware.
     status : StatusType, defaults to StatusType.SUCCESS
     starttime : float, defaults to 0.0
     endtime : float, defaults to 0.0
@@ -106,6 +108,7 @@ class TrialValue:
 
     cost: float | list[float]
     time: float = 0.0
+    cpu_time: float = 0.0
     status: StatusType = StatusType.SUCCESS
     starttime: float = 0.0
     endtime: float = 0.0

smac/runhistory/runhistory.py

@@ -173,6 +173,7 @@ def add(
         config: Configuration,
         cost: int | float | list[int | float],
         time: float = 0.0,
+        cpu_time: float = 0.0,
         status: StatusType = StatusType.SUCCESS,
         instance: str | None = None,
         seed: int | None = None,
@@ -191,6 +192,8 @@ def add(
             Cost of the evaluated trial. Might be a list in case of multi-objective.
         time : float
             How much time was needed to evaluate the trial.
+        cpu_time : float
+            How much time was needed on the hardware to evaluate the trial.
         status : StatusType, defaults to StatusType.SUCCESS
             The status of the trial.
         instance : str | None, defaults to none
@@ -254,6 +257,7 @@ def add(
         v = TrialValue(
             cost=c,
             time=time,
+            cpu_time=cpu_time,
             status=status,
             starttime=starttime,
             endtime=endtime,
@@ -269,6 +273,7 @@ def add(
             ("budget", budget),
             ("cost", c),
             ("time", time),
+            ("cpu_time", cpu_time),
             ("status", status),
             ("starttime", starttime),
             ("endtime", endtime),
@@ -310,6 +315,7 @@ def add_trial(self, info: TrialInfo, value: TrialValue) -> None:
             config=info.config,
             cost=value.cost,
             time=value.time,
+            cpu_time=value.cpu_time,
             status=value.status,
             instance=info.instance,
             seed=info.seed,
@@ -331,6 +337,7 @@ def add_running_trial(self, trial: TrialInfo) -> None:
             config=trial.config,
             cost=float(MAXINT),
             time=0.0,
+            cpu_time=0.0,
             status=StatusType.RUNNING,
             instance=trial.instance,
             seed=trial.seed,
@@ -771,6 +778,7 @@ def save(self, filename: str | Path = "runhistory.json") -> None:
                     float(k.budget) if k.budget is not None else None,
                     v.cost,
                     v.time,
+                    v.cpu_time,
                     v.status,
                     v.starttime,
                     v.endtime,
@@ -848,6 +856,7 @@ def load(self, filename: str | Path, configspace: ConfigurationSpace) -> None:
         self._n_id = len(self._config_ids)
 
         # Important to use add method to use all data structure correctly
+        # NOTE: These hardcoded indices can easily lead to trouble
         for entry in data["data"]:
             # Set n_objectives first
             if self._n_objectives == -1:
@@ -866,13 +875,14 @@ def load(self, filename: str | Path, configspace: ConfigurationSpace) -> None:
                 config=self._ids_config[int(entry[0])],
                 cost=cost,
                 time=float(entry[5]),
-                status=StatusType(entry[6]),
+                cpu_time=float(entry[6]),
+                status=StatusType(entry[7]),
                 instance=entry[1],
                 seed=entry[2],
                 budget=entry[3],
-                starttime=entry[7],
-                endtime=entry[8],
-                additional_info=entry[9],
+                starttime=entry[8],
+                endtime=entry[9],
+                additional_info=entry[10],
             )
 
         # Although adding trials should give us the same stats, the trajectory might be different
@@ -916,6 +926,7 @@ def update(self, runhistory: RunHistory) -> None:
                 config=config,
                 cost=value.cost,
                 time=value.time,
+                cpu_time=value.cpu_time,
                 status=value.status,
                 instance=key.instance,
                 starttime=value.starttime,

smac/runner/abstract_runner.py

@@ -105,9 +105,9 @@ def run_wrapper(
             Contains information about the status/performance of config.
         """
         start = time.time()
-
+        cpu_time = time.process_time()
         try:
-            status, cost, runtime, additional_info = self.run(
+            status, cost, runtime, cpu_time, additional_info = self.run(
                 config=trial_info.config,
                 instance=trial_info.instance,
                 budget=trial_info.budget,
@@ -117,6 +117,7 @@ def run_wrapper(
         except Exception as e:
             status = StatusType.CRASHED
             cost = self._crash_cost
+            cpu_time = time.process_time() - cpu_time
             runtime = time.time() - start
 
             # Add context information to the error message
@@ -148,6 +149,7 @@ def run_wrapper(
             status=status,
             cost=cost,
             time=runtime,
+            cpu_time=cpu_time,
             additional_info=additional_info,
             starttime=start,
             endtime=end,
@@ -185,7 +187,7 @@ def run(
         instance: str | None = None,
         budget: float | None = None,
         seed: int | None = None,
-    ) -> tuple[StatusType, float | list[float], float, dict]:
+    ) -> tuple[StatusType, float | list[float], float, float, dict]:
         """Runs the target function with a configuration on a single instance-budget-seed
         combination (aka trial).
 
@@ -208,6 +210,8 @@ def run(
             Resulting cost(s) of the trial.
         runtime : float
             The time the target function took to run.
+        cpu_time : float
+            The time the target function took on hardware to run.
         additional_info : dict
             All further additional trial information.
         """

smac/runner/dask_runner.py

@@ -163,7 +163,7 @@ def run(
         budget: float | None = None,
         seed: int | None = None,
         **dask_data_to_scatter: dict[str, Any],
-    ) -> tuple[StatusType, float | list[float], float, dict]:  # noqa: D102
+    ) -> tuple[StatusType, float | list[float], float, float, dict]:  # noqa: D102
         return self._single_worker.run(
             config=config, instance=instance, seed=seed, budget=budget, **dask_data_to_scatter
         )

smac/runner/target_function_runner.py

@@ -112,7 +112,7 @@ def run(
         budget: float | None = None,
         seed: int | None = None,
         **dask_data_to_scatter: dict[str, Any],
-    ) -> tuple[StatusType, float | list[float], float, dict]:
+    ) -> tuple[StatusType, float | list[float], float, float, dict]:
         """Calls the target function with pynisher if algorithm wall time limit or memory limit is
         set. Otherwise, the function is called directly.
 
@@ -143,6 +143,8 @@ def run(
             Resulting cost(s) of the trial.
         runtime : float
             The time the target function took to run.
+        cpu_time : float
+            The time the target function took on the hardware to run.
         additional_info : dict
             All further additional trial information.
         """
@@ -162,6 +164,7 @@ def run(
         # Presetting
         cost: float | list[float] = self._crash_cost
         runtime = 0.0
+        cpu_time = runtime
         additional_info = {}
         status = StatusType.CRASHED
 
@@ -183,7 +186,9 @@ def run(
         # Call target function
         try:
             start_time = time.time()
+            cpu_time = time.process_time()
             rval = self(config_copy, target_function, kwargs)
+            cpu_time = time.process_time() - cpu_time
             runtime = time.time() - start_time
             status = StatusType.SUCCESS
         except WallTimeoutException:
@@ -199,7 +204,7 @@ def run(
             status = StatusType.CRASHED
 
         if status != StatusType.SUCCESS:
-            return status, cost, runtime, additional_info
+            return status, cost, runtime, cpu_time, additional_info
 
         if isinstance(rval, tuple):
             result, additional_info = rval
@@ -240,7 +245,7 @@ def run(
         # We want to get either a float or a list of floats.
         cost = np.asarray(cost).squeeze().tolist()
 
-        return status, cost, runtime, additional_info
+        return status, cost, runtime, cpu_time, additional_info
 
     def __call__(
         self,

smac/runner/target_function_script_runner.py

@@ -83,7 +83,7 @@ def run(
         instance: str | None = None,
         budget: float | None = None,
         seed: int | None = None,
-    ) -> tuple[StatusType, float | list[float], float, dict]:
+    ) -> tuple[StatusType, float | list[float], float, float, dict]:
         """Calls the target function.
 
         Parameters
@@ -105,6 +105,8 @@ def run(
             Resulting cost(s) of the trial.
         runtime : float
             The time the target function took to run.
+        cpu_time : float
+            The time the target function took on the hardware to run.
         additional_info : dict
             All further additional trial information.
         """
@@ -128,6 +130,7 @@ def run(
         # Presetting
         cost: float | list[float] = self._crash_cost
         runtime = 0.0
+        cpu_time = runtime
         additional_info = {}
         status = StatusType.SUCCESS
 
@@ -139,7 +142,9 @@ def run(
 
         # Call target function
         start_time = time.time()
+        cpu_time = time.process_time()
         output, error = self(kwargs)
+        cpu_time = time.process_time() - cpu_time
         runtime = time.time() - start_time
 
         # Now we have to parse the std output
@@ -181,6 +186,10 @@ def run(
         if "runtime" in outputs:
             runtime = float(outputs["runtime"])
 
+        # Overwrite CPU time
+        if "cpu_time" in outputs:
+            cpu_time = float(outputs["cpu_time"])
+
         # Add additional info
         if "additional_info" in outputs:
             additional_info["additional_info"] = outputs["additional_info"]
@@ -194,7 +203,7 @@ def run(
                     "The target function crashed but returned a cost. The cost is ignored and replaced by crash cost."
                 )
 
-        return status, cost, runtime, additional_info
+        return status, cost, runtime, cpu_time, additional_info
 
     def __call__(
         self,