From ff308d496da3d883f115850dcf4bc96cdb46191b Mon Sep 17 00:00:00 2001 From: Antony Lewis Date: Thu, 22 Aug 2024 10:26:40 +0100 Subject: [PATCH 1/4] fixes for crash on chain write --- cobaya/collection.py | 13 +++++++------ cobaya/tools.py | 23 +++++++++++++++++++++++ 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/cobaya/collection.py b/cobaya/collection.py index 8da770d6c..400c0053c 100644 --- a/cobaya/collection.py +++ b/cobaya/collection.py @@ -96,7 +96,7 @@ def compute_temperature(logpost, logprior, loglike, check=True, extra_tolerance= """ Returns the temperature of a sample. - If ``check=True`` and the log-probabilites passed are arrays, checks consistency + If ``check=True`` and the log-probabilities passed are arrays, checks consistency of the sample temperature, and raises ``AssertionError`` if inconsistent. """ temp = (logprior + loglike) / logpost @@ -286,8 +286,7 @@ def __init__(self, model, output=None, cache_size=_default_cache_size, name=None self.reset() # If loaded, check sample weights, consistent logp sums, # and temperature (ignores the given one) - samples_loaded = len(self) > 0 - if samples_loaded: + if len(self) > 0: try: try: self.temperature = self._check_logps(extra_tolerance=False) @@ -497,10 +496,11 @@ def _check_logps(self, temperature_only=False, extra_tolerance=False): temperature = compute_temperature( -self["minuslogpost"], -self["minuslogprior"], -self["chi2"] * 0.5, check=True, extra_tolerance=extra_tolerance) - except AssertionError as excpt: + except AssertionError: raise LoggedError( - self.log, "The sample seems to have an inconsistent temperature.") \ - from excpt + self.log, "The sample seems to have an inconsistent temperature. " + "This could be due to input file truncation on the last line " + "due to crash/being killed before complete.") if not temperature_only: tols = { "rtol": 1e-4 * (10 if extra_tolerance else 1), @@ -1181,6 +1181,7 @@ def _dump_slice__txt(self, n_min=None, n_max=None): with open(self.file_name, "a", encoding="utf-8") as out: np.savetxt(out, self.data[n_min:n_max].to_numpy(dtype=np.float64), fmt=self._numpy_fmts) + os.fsync(out.fileno()) def _delete__txt(self): try: diff --git a/cobaya/tools.py b/cobaya/tools.py index 9cb21ead7..8c4887e0c 100644 --- a/cobaya/tools.py +++ b/cobaya/tools.py @@ -457,6 +457,19 @@ def read_dnumber(n: Any, dim: int): return NumberWithUnits(n, "d", dtype=int, scale=dim).value +def truncate_to_end_line(file_name): + with open(file_name, "r+b") as inp: + # Find the last complete line + inp.seek(0, 2) # Go to the end of the file + pos = inp.tell() - 1 + while pos > 0 and inp.read(1) != "\n": + pos -= 1 + inp.seek(pos, 0) + if pos > 0: + inp.seek(pos + 1, 0) + inp.truncate() + + def load_DataFrame(file_name, skip=0, root_file_name=None): """ Loads a `pandas.DataFrame` from a text file @@ -493,6 +506,16 @@ def load_DataFrame(file_name, skip=0, root_file_name=None): inp, sep=" ", header=None, names=cols, comment="#", skipinitialspace=True, skiprows=skip, index_col=False) + if not data.empty: + # Check if the last row contains any NaNs + if data.iloc[-1].isna().any(): + log.warning("Last row of %s is incomplete or contains NaNs", file_name) + # If the second-to-last row exists and doesn't contain NaNs, + # delete the last row assuming this was due to crash on write + if len(data) > 1 and not data.iloc[-2].isna().any(): + data = data.iloc[:-1] + log.info(f"Saving {file_name} deleting last (in)complete line") + truncate_to_end_line(file_name) return data From f7004021824ab4a12f2b13c43ed3a84b03511fde Mon Sep 17 00:00:00 2001 From: Antony Lewis Date: Thu, 22 Aug 2024 10:32:22 +0100 Subject: [PATCH 2/4] fix --- cobaya/collection.py | 4 ++-- cobaya/tools.py | 22 +++++++++++----------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/cobaya/collection.py b/cobaya/collection.py index 400c0053c..4d717a94f 100644 --- a/cobaya/collection.py +++ b/cobaya/collection.py @@ -496,11 +496,11 @@ def _check_logps(self, temperature_only=False, extra_tolerance=False): temperature = compute_temperature( -self["minuslogpost"], -self["minuslogprior"], -self["chi2"] * 0.5, check=True, extra_tolerance=extra_tolerance) - except AssertionError: + except AssertionError as excpt: raise LoggedError( self.log, "The sample seems to have an inconsistent temperature. " "This could be due to input file truncation on the last line " - "due to crash/being killed before complete.") + "due to crash/being killed before complete.") from excpt if not temperature_only: tols = { "rtol": 1e-4 * (10 if extra_tolerance else 1), diff --git a/cobaya/tools.py b/cobaya/tools.py index 8c4887e0c..d15839abb 100644 --- a/cobaya/tools.py +++ b/cobaya/tools.py @@ -506,17 +506,17 @@ def load_DataFrame(file_name, skip=0, root_file_name=None): inp, sep=" ", header=None, names=cols, comment="#", skipinitialspace=True, skiprows=skip, index_col=False) - if not data.empty: - # Check if the last row contains any NaNs - if data.iloc[-1].isna().any(): - log.warning("Last row of %s is incomplete or contains NaNs", file_name) - # If the second-to-last row exists and doesn't contain NaNs, - # delete the last row assuming this was due to crash on write - if len(data) > 1 and not data.iloc[-2].isna().any(): - data = data.iloc[:-1] - log.info(f"Saving {file_name} deleting last (in)complete line") - truncate_to_end_line(file_name) - return data + if not data.empty: + # Check if the last row contains any NaNs + if data.iloc[-1].isna().any(): + log.warning("Last row of %s is incomplete or contains NaNs", file_name) + # If the second-to-last row exists and doesn't contain NaNs, + # delete the last row assuming this was due to crash on write + if len(data) > 1 and not data.iloc[-2].isna().any(): + data = data.iloc[:-1] + log.info(f"Saving {file_name} deleting last (in)complete line") + truncate_to_end_line(file_name) + return data def prepare_comment(comment): From ec80b0797697c88c18eb90e9ae523a14bf883673 Mon Sep 17 00:00:00 2001 From: Antony Lewis Date: Thu, 22 Aug 2024 10:44:45 +0100 Subject: [PATCH 3/4] fix --- cobaya/tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cobaya/tools.py b/cobaya/tools.py index d15839abb..903ce6a01 100644 --- a/cobaya/tools.py +++ b/cobaya/tools.py @@ -462,7 +462,7 @@ def truncate_to_end_line(file_name): # Find the last complete line inp.seek(0, 2) # Go to the end of the file pos = inp.tell() - 1 - while pos > 0 and inp.read(1) != "\n": + while pos > 0 and inp.read(1) != b"\n": pos -= 1 inp.seek(pos, 0) if pos > 0: From 4f7ad49d24081c7ebe5e02d8422afbbe055b8024 Mon Sep 17 00:00:00 2001 From: Antony Lewis Date: Thu, 22 Aug 2024 13:56:41 +0100 Subject: [PATCH 4/4] fix --- cobaya/collection.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cobaya/collection.py b/cobaya/collection.py index 4d717a94f..e0c5d11ca 100644 --- a/cobaya/collection.py +++ b/cobaya/collection.py @@ -1181,7 +1181,6 @@ def _dump_slice__txt(self, n_min=None, n_max=None): with open(self.file_name, "a", encoding="utf-8") as out: np.savetxt(out, self.data[n_min:n_max].to_numpy(dtype=np.float64), fmt=self._numpy_fmts) - os.fsync(out.fileno()) def _delete__txt(self): try: