From cb34ebd101eebb99b6f9cbacd31ca9790f6ca5be Mon Sep 17 00:00:00 2001 From: Daniel Weindl Date: Thu, 5 Dec 2024 23:11:11 +0100 Subject: [PATCH] v2: Adapt to long conditions table Update creation of PEtab problems, validation, conversion from v1 to v2,... to the new long condition table. --- petab/v1/__init__.py | 2 + petab/v1/lint.py | 2 +- petab/v2/C.py | 3 +- petab/v2/__init__.py | 34 +++-- petab/v2/_helpers.py | 2 + petab/v2/conditions.py | 67 ++++++++++ petab/v2/lint.py | 255 +++++++++++++++++++++++++++++------- petab/v2/petab1to2.py | 125 ++++++++++++++---- petab/v2/problem.py | 42 ++++-- tests/v2/test_conversion.py | 33 +++-- tests/v2/test_problem.py | 19 ++- tox.ini | 2 +- 12 files changed, 472 insertions(+), 114 deletions(-) create mode 100644 petab/v2/_helpers.py create mode 100644 petab/v2/conditions.py diff --git a/petab/v1/__init__.py b/petab/v1/__init__.py index a8609621..cd21b88a 100644 --- a/petab/v1/__init__.py +++ b/petab/v1/__init__.py @@ -4,6 +4,7 @@ """ from ..version import __version__ # noqa: F401, E402 +from . import models # noqa: F401, E402 from .C import * # noqa: F403, F401, E402 from .calculate import * # noqa: F403, F401, E402 from .composite_problem import * # noqa: F403, F401, E402 @@ -13,6 +14,7 @@ from .lint import * # noqa: F403, F401, E402 from .mapping import * # noqa: F403, F401, E402 from .measurements import * # noqa: F403, F401, E402 +from .models import Model # noqa: F401, E402 from .observables import * # noqa: F403, F401, E402 from .parameter_mapping import * # noqa: F403, F401, E402 from .parameters import * # noqa: F403, F401, E402 diff --git a/petab/v1/lint.py b/petab/v1/lint.py index 6f70520b..e970bfde 100644 --- a/petab/v1/lint.py +++ b/petab/v1/lint.py @@ -67,7 +67,7 @@ def _check_df(df: pd.DataFrame, req_cols: Iterable, name: str) -> None: """ if missing_cols := set(req_cols) - set(df.columns.values): raise AssertionError( - f"DataFrame {name} requires the columns {missing_cols}." + f"{name.capitalize()} table requires the columns {missing_cols}." ) diff --git a/petab/v2/C.py b/petab/v2/C.py index 2d55355a..cb095c68 100644 --- a/petab/v2/C.py +++ b/petab/v2/C.py @@ -136,7 +136,6 @@ # TODO: removed? #: Condition name column in the condition table CONDITION_NAME = "conditionName" - #: Column in the condition table with the ID of an entity that is changed TARGET_ID = "targetId" #: Column in the condition table with the type of value that is changed @@ -166,6 +165,8 @@ TARGET_VALUE, ] +CONDITION_DF_REQUIRED_COLS = CONDITION_DF_COLS + # EXPERIMENTS EXPERIMENT_DF_REQUIRED_COLS = [ EXPERIMENT_ID, diff --git a/petab/v2/__init__.py b/petab/v2/__init__.py index ca55f7d0..0525d66c 100644 --- a/petab/v2/__init__.py +++ b/petab/v2/__init__.py @@ -4,17 +4,35 @@ """ from warnings import warn -from ..v1 import * # noqa: F403, F401, E402 -from .experiments import ( # noqa: F401 - get_experiment_df, - write_experiment_df, -) - -# import after v1 -from .problem import Problem # noqa: F401 +# TODO: remove v1 star imports +from ..v1.calculate import * # noqa: F403, F401, E402 +from ..v1.composite_problem import * # noqa: F403, F401, E402 +from ..v1.core import * # noqa: F403, F401, E402 +from ..v1.format_version import __format_version__ # noqa: F401, E402 +from ..v1.mapping import * # noqa: F403, F401, E402 +from ..v1.measurements import * # noqa: F403, F401, E402 +from ..v1.observables import * # noqa: F403, F401, E402 +from ..v1.parameter_mapping import * # noqa: F403, F401, E402 +from ..v1.parameters import * # noqa: F403, F401, E402 +from ..v1.sampling import * # noqa: F403, F401, E402 +from ..v1.sbml import * # noqa: F403, F401, E402 +from ..v1.simulate import * # noqa: F403, F401, E402 +from ..v1.yaml import * # noqa: F403, F401, E402 warn( "Support for PEtab2.0 and all of petab.v2 is experimental " "and subject to changes!", stacklevel=1, ) + +# import after v1 +from ..version import __version__ # noqa: F401, E402 +from . import models # noqa: F401, E402 +from .conditions import * # noqa: F403, F401, E402 +from .experiments import ( # noqa: F401, E402 + get_experiment_df, + write_experiment_df, +) +from .lint import lint_problem # noqa: F401, E402 +from .models import Model # noqa: F401, E402 +from .problem import Problem # noqa: F401, E402 diff --git a/petab/v2/_helpers.py b/petab/v2/_helpers.py new file mode 100644 index 00000000..a7522f35 --- /dev/null +++ b/petab/v2/_helpers.py @@ -0,0 +1,2 @@ +"""Various internal helper functions.""" +from ..v1.core import to_float_if_float # noqa: F401, E402 diff --git a/petab/v2/conditions.py b/petab/v2/conditions.py new file mode 100644 index 00000000..7bb6d262 --- /dev/null +++ b/petab/v2/conditions.py @@ -0,0 +1,67 @@ +"""Functions operating on the PEtab condition table""" +from __future__ import annotations + +from pathlib import Path + +import pandas as pd +import sympy as sp + +from .. import v2 +from ..v1.math import sympify_petab +from .C import * +from .lint import assert_no_leading_trailing_whitespace + +__all__ = [ + "get_condition_df", + "write_condition_df", +] + + +def get_condition_df( + condition_file: str | pd.DataFrame | Path | None, +) -> pd.DataFrame | None: + """Read the provided condition file into a ``pandas.Dataframe``. + + Arguments: + condition_file: File name of PEtab condition file or pandas.Dataframe + """ + if condition_file is None: + return condition_file + + if isinstance(condition_file, str | Path): + condition_file = pd.read_csv( + condition_file, sep="\t", float_precision="round_trip" + ) + + assert_no_leading_trailing_whitespace( + condition_file.columns.values, "condition" + ) + + return condition_file + + +def write_condition_df(df: pd.DataFrame, filename: str | Path) -> None: + """Write PEtab condition table + + Arguments: + df: PEtab condition table + filename: Destination file name + """ + df = get_condition_df(df) + df.to_csv(filename, sep="\t", index=False) + + +def get_condition_table_free_symbols(problem: v2.Problem) -> set[sp.Basic]: + """Free symbols from condition table assignments. + + Collects all free symbols from the condition table `targetValue` column. + + :returns: Set of free symbols. + """ + if problem.condition_df is None: + return set() + + free_symbols = set() + for target_value in problem.condition_df[TARGET_VALUE]: + free_symbols |= sympify_petab(target_value).free_symbols + return free_symbols diff --git a/petab/v2/lint.py b/petab/v2/lint.py index fdf6de0c..d27e6fda 100644 --- a/petab/v2/lint.py +++ b/petab/v2/lint.py @@ -3,6 +3,7 @@ import logging from abc import ABC, abstractmethod +from collections import OrderedDict from dataclasses import dataclass, field from enum import IntEnum from pathlib import Path @@ -10,9 +11,10 @@ import numpy as np import pandas as pd -from petab.v1.conditions import get_parametric_overrides -from petab.v1.lint import ( +from .. import v2 +from ..v1.lint import ( _check_df, + assert_model_parameters_in_condition_or_parameter_table, assert_no_leading_trailing_whitespace, assert_parameter_bounds_are_numeric, assert_parameter_estimate_is_boolean, @@ -22,25 +24,14 @@ assert_parameter_scale_is_valid, assert_unique_parameter_ids, check_ids, - check_parameter_bounds, -) -from petab.v1.measurements import split_parameter_replacement_list -from petab.v1.observables import get_output_parameters, get_placeholders -from petab.v1.parameters import ( - get_valid_parameters_for_parameter_table, -) -from petab.v1.visualize.lint import validate_visualization_df -from petab.v2 import ( - assert_model_parameters_in_condition_or_parameter_table, -) -from petab.v2.C import * - -from ..v1 import ( - assert_measurement_conditions_present_in_condition_table, - check_condition_df, check_measurement_df, check_observable_df, + check_parameter_bounds, ) +from ..v1.measurements import split_parameter_replacement_list +from ..v1.observables import get_output_parameters, get_placeholders +from ..v1.visualize.lint import validate_visualization_df +from ..v2.C import * from .problem import Problem logger = logging.getLogger(__name__) @@ -247,15 +238,55 @@ def run(self, problem: Problem) -> ValidationIssue | None: try: check_measurement_df(problem.measurement_df, problem.observable_df) - - if problem.condition_df is not None: - # TODO: handle missing condition_df - assert_measurement_conditions_present_in_condition_table( - problem.measurement_df, problem.condition_df - ) except AssertionError as e: return ValidationError(str(e)) + # TODO: introduce some option for validation partial vs full + # problem. if this is supposed to be a complete problem, a missing + # condition table should be an error if the measurement table refers + # to conditions + + # check that measured experiments/conditions exist + # TODO: fully switch to experiment table and remove this: + if SIMULATION_CONDITION_ID in problem.measurement_df: + if problem.condition_df is None: + return + used_conditions = set( + problem.measurement_df[SIMULATION_CONDITION_ID].dropna().values + ) + if PREEQUILIBRATION_CONDITION_ID in problem.measurement_df: + used_conditions |= set( + problem.measurement_df[PREEQUILIBRATION_CONDITION_ID] + .dropna() + .values + ) + available_conditions = set( + problem.condition_df[CONDITION_ID].unique() + ) + if missing_conditions := (used_conditions - available_conditions): + return ValidationError( + "Measurement table references conditions that " + "are not specified in the condition table: " + + str(missing_conditions) + ) + elif EXPERIMENT_ID in problem.measurement_df: + if problem.experiment_df is None: + return + used_experiments = set( + problem.measurement_df[EXPERIMENT_ID].values + ) + available_experiments = set( + problem.condition_df[CONDITION_ID].unique() + ) + if missing_experiments := ( + used_experiments - available_experiments + ): + raise AssertionError( + "Measurement table references experiments that " + "are not specified in the experiments table: " + + str(missing_experiments) + ) + class CheckConditionTable(ValidationTask): """A task to validate the condition table of a PEtab problem.""" @@ -264,16 +295,42 @@ def run(self, problem: Problem) -> ValidationIssue | None: if problem.condition_df is None: return + df = problem.condition_df + try: - check_condition_df( - problem.condition_df, - model=problem.model, - observable_df=problem.observable_df, - mapping_df=problem.mapping_df, - ) + _check_df(df, CONDITION_DF_REQUIRED_COLS, "condition") + check_ids(df[CONDITION_ID], kind="condition") + check_ids(df[TARGET_ID], kind="target") except AssertionError as e: return ValidationError(str(e)) + # TODO: check value types + + if problem.model is None: + return + + # check targets are valid + allowed_targets = set( + problem.model.get_valid_ids_for_condition_table() + ) + if problem.observable_df is not None: + allowed_targets |= set( + get_output_parameters( + model=problem.model, + observable_df=problem.observable_df, + mapping_df=problem.mapping_df, + ) + ) + if problem.mapping_df is not None: + allowed_targets |= set(problem.mapping_df.index.values) + invalid = set(df[TARGET_ID].unique()) - allowed_targets + if invalid: + return ValidationError( + f"Condition table contains invalid targets: {invalid}" + ) + + # TODO check that all value types are valid for the given targets + class CheckObservableTable(ValidationTask): """A task to validate the observable table of a PEtab problem.""" @@ -454,14 +511,7 @@ def run(self, problem: Problem) -> ValidationIssue | None: return required = get_required_parameters_for_parameter_table(problem) - - allowed = get_valid_parameters_for_parameter_table( - model=problem.model, - condition_df=problem.condition_df, - observable_df=problem.observable_df, - measurement_df=problem.measurement_df, - mapping_df=problem.mapping_df, - ) + allowed = get_valid_parameters_for_parameter_table(problem) actual = set(problem.parameter_df.index) missing = required - actual @@ -542,6 +592,102 @@ def run(self, problem: Problem) -> ValidationIssue | None: ) +def get_valid_parameters_for_parameter_table( + problem: Problem, +) -> set[str]: + """ + Get set of parameters which may be present inside the parameter table + + Arguments: + model: PEtab model + condition_df: PEtab condition table + observable_df: PEtab observable table + measurement_df: PEtab measurement table + mapping_df: PEtab mapping table for additional checks + + Returns: + Set of parameter IDs which PEtab allows to be present in the + parameter table. + """ + # - grab all allowed model parameters + # - grab corresponding names from mapping table + # - grab all output parameters defined in {observable,noise}Formula + # - grab all parameters from measurement table + # - grab all parametric overrides from condition table + # - remove parameters for which condition table columns exist + # - remove placeholder parameters + # (only partial overrides are not supported) + model = problem.model + condition_df = problem.condition_df + observable_df = problem.observable_df + measurement_df = problem.measurement_df + mapping_df = problem.mapping_df + + # must not go into parameter table + blackset = set() + + if observable_df is not None: + placeholders = set(get_placeholders(observable_df)) + + # collect assignment targets + blackset |= placeholders + + if condition_df is not None: + blackset |= set(condition_df.columns.values) - {CONDITION_NAME} + + # don't use sets here, to have deterministic ordering, + # e.g. for creating parameter tables + parameter_ids = OrderedDict.fromkeys( + p + for p in model.get_valid_parameters_for_parameter_table() + if p not in blackset + ) + + if mapping_df is not None: + for from_id, to_id in zip( + mapping_df.index.values, mapping_df[MODEL_ENTITY_ID], strict=True + ): + if to_id in parameter_ids.keys(): + parameter_ids[from_id] = None + + if observable_df is not None: + # add output parameters from observables table + output_parameters = get_output_parameters( + observable_df=observable_df, model=model + ) + for p in output_parameters: + if p not in blackset: + parameter_ids[p] = None + + # Append parameters from measurement table, unless they occur as condition + # table columns + def append_overrides(overrides): + for p in overrides: + if isinstance(p, str) and p not in blackset: + parameter_ids[p] = None + + if measurement_df is not None: + for _, row in measurement_df.iterrows(): + # we trust that the number of overrides matches + append_overrides( + split_parameter_replacement_list( + row.get(OBSERVABLE_PARAMETERS, None) + ) + ) + append_overrides( + split_parameter_replacement_list( + row.get(NOISE_PARAMETERS, None) + ) + ) + + # Append parameter overrides from condition table + if condition_df is not None: + for p in v2.conditions.get_condition_table_free_symbols(problem): + parameter_ids[str(p)] = None + + return parameter_ids.keys() + + def get_required_parameters_for_parameter_table( problem: Problem, ) -> set[str]: @@ -563,7 +709,11 @@ def append_overrides(overrides): parameter_ids.update( p for p in overrides - if isinstance(p, str) and p not in problem.condition_df.columns + if isinstance(p, str) + and ( + problem.condition_df is None + or p not in problem.condition_df[TARGET_ID] + ) ) for _, row in problem.measurement_df.iterrows(): @@ -616,20 +766,24 @@ def append_overrides(overrides): # Add condition table parametric overrides unless already defined in the # model parameter_ids.update( - p - for p in get_parametric_overrides(problem.condition_df) - if not problem.model.has_entity_with_id(p) + str(p) + for p in v2.conditions.get_condition_table_free_symbols(problem) + if not problem.model.has_entity_with_id(str(p)) ) # remove parameters that occur in the condition table and are overridden # for ALL conditions - for p in problem.condition_df.columns[ - ~problem.condition_df.isnull().any() - ]: - try: - parameter_ids.remove(p) - except KeyError: - pass + if problem.condition_df is not None: + ... + # FIXME: update to v2 -- we need to check for each value type + # separately?! + # for p in problem.condition_df.columns[ + # ~problem.condition_df.isnull().any() + # ]: + # try: + # parameter_ids.remove(p) + # except KeyError: + # pass return parameter_ids @@ -650,6 +804,7 @@ def append_overrides(overrides): CheckObservablesDoNotShadowModelEntities(), CheckParameterTable(), CheckAllParametersPresentInParameterTable(), - CheckVisualizationTable(), + # TODO: atomize checks, update to long condition table, re-enable + # CheckVisualizationTable(), CheckValidParameterInConditionOrParameterTable(), ] diff --git a/petab/v2/petab1to2.py b/petab/v2/petab1to2.py index 866414c3..7eb33dc7 100644 --- a/petab/v2/petab1to2.py +++ b/petab/v2/petab1to2.py @@ -1,18 +1,20 @@ """Convert PEtab version 1 problems to version 2.""" import shutil +from contextlib import suppress from itertools import chain from pathlib import Path from urllib.parse import urlparse +import numpy as np +import pandas as pd from pandas.io.common import get_handle, is_url -import petab.v1.C as C +import petab.v1.C from petab.models import MODEL_TYPE_SBML from petab.v1 import Problem as ProblemV1 -from petab.v2.lint import lint_problem as lint_v2_problem from petab.yaml import get_path_prefix -from ..v1 import lint_problem as lint_v1_problem +from .. import v1, v2 from ..v1.yaml import load_yaml, validate, write_yaml from ..versions import get_major_version @@ -61,8 +63,8 @@ def petab1to2(yaml_config: Path | str, output_dir: Path | str = None): if get_major_version(yaml_config) != 1: raise ValueError("PEtab problem is not version 1.") petab_problem = ProblemV1.from_yaml(yaml_file or yaml_config) - if lint_v1_problem(petab_problem): - raise ValueError("PEtab problem does not pass linting.") + if v1.lint_problem(petab_problem): + raise ValueError("Provided PEtab problem does not pass linting.") # Update YAML file new_yaml_config = _update_yaml(yaml_config) @@ -76,28 +78,57 @@ def petab1to2(yaml_config: Path | str, output_dir: Path | str = None): # Update tables # condition tables, observable tables, SBML files, parameter table: # no changes - just copy - file = yaml_config[C.PARAMETER_FILE] + file = yaml_config[v2.C.PARAMETER_FILE] _copy_file(get_src_path(file), Path(get_dest_path(file))) - for problem_config in yaml_config[C.PROBLEMS]: + for problem_config in yaml_config[v2.C.PROBLEMS]: for file in chain( - problem_config.get(C.CONDITION_FILES, []), - problem_config.get(C.OBSERVABLE_FILES, []), + problem_config.get(v2.C.OBSERVABLE_FILES, []), ( - model[C.MODEL_LOCATION] - for model in problem_config.get(C.MODEL_FILES, {}).values() + model[v2.C.MODEL_LOCATION] + for model in problem_config.get(v2.C.MODEL_FILES, {}).values() ), - problem_config.get(C.MEASUREMENT_FILES, []), - problem_config.get(C.VISUALIZATION_FILES, []), + problem_config.get(v2.C.VISUALIZATION_FILES, []), ): _copy_file(get_src_path(file), Path(get_dest_path(file))) + # Update condition table + for condition_file in problem_config.get(v2.C.CONDITION_FILES, []): + condition_df = v1.get_condition_df(get_src_path(condition_file)) + condition_df = _melt_condition_df( + condition_df, petab_problem.model + ) + v2.write_condition_df(condition_df, get_dest_path(condition_file)) + + for measurement_file in problem_config.get(v2.C.MEASUREMENT_FILES, []): + measurement_df = v1.get_measurement_df( + get_src_path(measurement_file) + ) + if ( + petab_problem.condition_df is not None + and len( + set(petab_problem.condition_df.columns) + - {petab.v1.C.CONDITION_NAME} + ) + == 0 + ): + # can't have "empty" conditions with no overrides in v2 + # TODO: this needs to be done condition wise + measurement_df[v2.C.SIMULATION_CONDITION_ID] = np.nan + if ( + v1.C.PREEQUILIBRATION_CONDITION_ID + in measurement_df.columns + ): + measurement_df[v2.C.PREEQUILIBRATION_CONDITION_ID] = np.nan + v2.write_measurement_df( + measurement_df, get_dest_path(measurement_file) + ) # TODO: Measurements: preequilibration to experiments/timecourses once # finalized ... # validate updated Problem - validation_issues = lint_v2_problem(new_yaml_file) + validation_issues = v2.lint_problem(new_yaml_file) if validation_issues: raise ValueError( @@ -111,23 +142,23 @@ def _update_yaml(yaml_config: dict) -> dict: yaml_config = yaml_config.copy() # Update format_version - yaml_config[C.FORMAT_VERSION] = "2.0.0" + yaml_config[v2.C.FORMAT_VERSION] = "2.0.0" # Add extensions - yaml_config[C.EXTENSIONS] = [] + yaml_config[v2.C.EXTENSIONS] = [] # Move models and set IDs (filename for now) - for problem in yaml_config[C.PROBLEMS]: - problem[C.MODEL_FILES] = {} - models = problem[C.MODEL_FILES] - for sbml_file in problem[C.SBML_FILES]: + for problem in yaml_config[v2.C.PROBLEMS]: + problem[v2.C.MODEL_FILES] = {} + models = problem[v2.C.MODEL_FILES] + for sbml_file in problem[v1.C.SBML_FILES]: model_id = sbml_file.split("/")[-1].split(".")[0] models[model_id] = { - C.MODEL_LANGUAGE: MODEL_TYPE_SBML, - C.MODEL_LOCATION: sbml_file, + v2.C.MODEL_LANGUAGE: MODEL_TYPE_SBML, + v2.C.MODEL_LOCATION: sbml_file, } - problem[C.MODEL_FILES] = problem.get(C.MODEL_FILES, {}) - del problem[C.SBML_FILES] + problem[v2.C.MODEL_FILES] = problem.get(v2.C.MODEL_FILES, {}) + del problem[v1.C.SBML_FILES] return yaml_config @@ -152,3 +183,49 @@ def _copy_file(src: Path | str, dest: Path): return except FileNotFoundError: shutil.copy(str(src), str(dest)) + + +def _melt_condition_df( + condition_df: pd.DataFrame, model: v1.Model +) -> pd.DataFrame: + """Melt condition table.""" + condition_df = condition_df.copy().reset_index() + with suppress(KeyError): + # TODO: are condition names still supported in v2? + condition_df.drop(columns=[v2.C.CONDITION_NAME], inplace=True) + + condition_df = condition_df.melt( + id_vars=[v1.C.CONDITION_ID], + var_name=v2.C.TARGET_ID, + value_name=v2.C.TARGET_VALUE, + ) + + if condition_df.empty: + # This happens if there weren't any condition-specific changes + return pd.DataFrame( + columns=[ + v2.C.CONDITION_ID, + v2.C.TARGET_ID, + v2.C.VALUE_TYPE, + v2.C.TARGET_VALUE, + ] + ) + + targets = set(condition_df[v2.C.TARGET_ID].unique()) + valid_cond_pars = set(model.get_valid_parameters_for_parameter_table()) + # entities to which we assign constant values + constant = targets & valid_cond_pars + # entities to which we assign initial values + initial = set() + for target in targets - constant: + if model.is_state_variable(target): + initial.add(target) + else: + raise NotImplementedError( + f"Unable to determine value type {target} in the condition " + "table." + ) + condition_df[v2.C.VALUE_TYPE] = condition_df[v2.C.TARGET_ID].apply( + lambda x: v2.C.VT_INITIAL if x in initial else v2.C.VT_CONSTANT + ) + return condition_df diff --git a/petab/v2/problem.py b/petab/v2/problem.py index b61d8b14..1df2c677 100644 --- a/petab/v2/problem.py +++ b/petab/v2/problem.py @@ -4,6 +4,7 @@ import logging import os import tempfile +import traceback import warnings from collections.abc import Sequence from math import nan @@ -15,7 +16,6 @@ from pydantic import AnyUrl, BaseModel, Field from ..v1 import ( - conditions, core, mapping, measurements, @@ -29,10 +29,10 @@ from ..v1.problem import ListOfFiles, VersionNumber from ..v1.yaml import get_path_prefix from ..v2.C import * # noqa: F403 -from . import experiments +from . import conditions, experiments if TYPE_CHECKING: - from ..v2.lint import ValidationIssue, ValidationResultList, ValidationTask + from ..v2.lint import ValidationResultList, ValidationTask __all__ = ["Problem"] @@ -722,7 +722,11 @@ def validate( Returns: A list of validation results. """ - from ..v2.lint import ValidationIssueSeverity, ValidationResultList + from ..v2.lint import ( + ValidationIssue, + ValidationIssueSeverity, + ValidationResultList, + ) validation_results = ValidationResultList() if self.extensions_config: @@ -741,7 +745,8 @@ def validate( except Exception as e: cur_result = ValidationIssue( ValidationIssueSeverity.CRITICAL, - f"Validation task {task} failed with exception: {e}", + f"Validation task {task} failed with exception: {e}\n" + f"{traceback.format_exc()}", ) if cur_result: @@ -752,20 +757,35 @@ def validate( return validation_results - def add_condition(self, id_: str, name: str = None, **kwargs): + def add_condition( + self, id_: str, name: str = None, **kwargs: tuple[str, Number | str] + ): """Add a simulation condition to the problem. Arguments: id_: The condition id name: The condition name - kwargs: Parameter, value pairs to add to the condition table. + kwargs: Entities to be added to the condition table in the form + `target_id=(value_type, target_value)`. """ - record = {CONDITION_ID: [id_], **kwargs} + if not kwargs: + return + records = [ + { + CONDITION_ID: id_, + TARGET_ID: target_id, + VALUE_TYPE: value_type, + TARGET_VALUE: target_value, + } + for target_id, (value_type, target_value) in kwargs.items() + ] + # TODO: is the condition name supported in v2? if name is not None: - record[CONDITION_NAME] = name - tmp_df = pd.DataFrame(record).set_index([CONDITION_ID]) + for record in records: + record[CONDITION_NAME] = [name] + tmp_df = pd.DataFrame(records) self.condition_df = ( - pd.concat([self.condition_df, tmp_df]) + pd.concat([self.condition_df, tmp_df], ignore_index=True) if self.condition_df is not None else tmp_df ) diff --git a/tests/v2/test_conversion.py b/tests/v2/test_conversion.py index c309a00e..4b982fcf 100644 --- a/tests/v2/test_conversion.py +++ b/tests/v2/test_conversion.py @@ -1,6 +1,8 @@ import logging import tempfile +import pytest + from petab.v2.petab1to2 import petab1to2 @@ -16,19 +18,28 @@ def test_petab1to2_remote(): petab1to2(yaml_url, tmpdirname) -def test_benchmark_collection(): - """Test that we can upgrade all benchmark collection models.""" +try: import benchmark_models_petab + parametrize_or_skip = pytest.mark.parametrize( + "problem_id", benchmark_models_petab.MODELS + ) +except ImportError: + parametrize_or_skip = pytest.mark.skip( + reason="benchmark_models_petab not installed" + ) + + +@parametrize_or_skip +def test_benchmark_collection(problem_id): + """Test that we can upgrade all benchmark collection models.""" logging.basicConfig(level=logging.DEBUG) - for problem_id in benchmark_models_petab.MODELS: - if problem_id == "Lang_PLOSComputBiol2024": - # Does not pass initial linting - continue + if problem_id == "Froehlich_CellSystems2018": + pytest.skip("Too slow. Re-enable once we are faster.") - yaml_path = benchmark_models_petab.get_problem_yaml_path(problem_id) - with tempfile.TemporaryDirectory( - prefix=f"test_petab1to2_{problem_id}" - ) as tmpdirname: - petab1to2(yaml_path, tmpdirname) + yaml_path = benchmark_models_petab.get_problem_yaml_path(problem_id) + with tempfile.TemporaryDirectory( + prefix=f"test_petab1to2_{problem_id}" + ) as tmpdirname: + petab1to2(yaml_path, tmpdirname) diff --git a/tests/v2/test_problem.py b/tests/v2/test_problem.py index 41ecc238..ba210af0 100644 --- a/tests/v2/test_problem.py +++ b/tests/v2/test_problem.py @@ -18,7 +18,11 @@ OBSERVABLE_ID, PARAMETER_ID, PETAB_ENTITY_ID, + TARGET_ID, + TARGET_VALUE, UPPER_BOUND, + VALUE_TYPE, + VT_CONSTANT, ) @@ -26,7 +30,7 @@ def test_load_remote(): """Test loading remote files""" yaml_url = ( "https://raw.githubusercontent.com/PEtab-dev/petab_test_suite" - "/main/petabtests/cases/v2.0.0/sbml/0001/_0001.yaml" + "/update_v2/petabtests/cases/v2.0.0/sbml/0001/_0001.yaml" ) petab_problem = Problem.from_yaml(yaml_url) @@ -69,7 +73,7 @@ def test_problem_from_yaml_multiple_files(): for i in (1, 2): problem = Problem() - problem.add_condition(f"condition{i}") + problem.add_condition(f"condition{i}", parameter1=(VT_CONSTANT, i)) petab.write_condition_df( problem.condition_df, Path(tmpdir, f"conditions{i}.tsv") ) @@ -105,16 +109,17 @@ def test_problem_from_yaml_multiple_files(): def test_modify_problem(): """Test modifying a problem via the API.""" problem = Problem() - problem.add_condition("condition1", parameter1=1) - problem.add_condition("condition2", parameter2=2) + problem.add_condition("condition1", parameter1=(VT_CONSTANT, 1)) + problem.add_condition("condition2", parameter2=(VT_CONSTANT, 2)) exp_condition_df = pd.DataFrame( data={ CONDITION_ID: ["condition1", "condition2"], - "parameter1": [1.0, np.nan], - "parameter2": [np.nan, 2.0], + TARGET_ID: ["parameter1", "parameter2"], + VALUE_TYPE: [VT_CONSTANT, VT_CONSTANT], + TARGET_VALUE: [1.0, 2.0], } - ).set_index([CONDITION_ID]) + ) assert_frame_equal( problem.condition_df, exp_condition_df, check_dtype=False ) diff --git a/tox.ini b/tox.ini index d57aa91d..7d0cdccc 100644 --- a/tox.ini +++ b/tox.ini @@ -19,7 +19,7 @@ deps= commands = python -m pip install sympy>=1.12.1 - python -m pytest --cov=petab --cov-report=xml --cov-append \ + python -m pytest --cov=petab --cov-report=xml --cov-append --durations=10 \ tests description = Basic tests