Validate all errors v2

chaoslib/activity.py

@@ -16,14 +16,16 @@
     validate_python_activity
 from chaoslib.provider.process import run_process_activity, \
     validate_process_activity
-from chaoslib.types import Activity, Configuration, Experiment, Run, Secrets
+from chaoslib.types import Activity, Configuration, Experiment, Run, Secrets, \
+    ValidationError
+from chaoslib.validation import Validation
 
 
 __all__ = ["ensure_activity_is_valid", "get_all_activities_in_experiment",
            "run_activities"]
 
 
-def ensure_activity_is_valid(activity: Activity):
+def ensure_activity_is_valid(activity: Activity) -> List[ValidationError]:
     """
     Goes through the activity and checks certain of its properties and raise
     :exc:`InvalidActivity` whenever one does not respect the expectations.
@@ -34,69 +36,80 @@ def ensure_activity_is_valid(activity: Activity):
 
     Depending on the type, an activity requires a variety of other keys.
 
-    In all failing cases, raises :exc:`InvalidActivity`.
+    In all failing cases, returns a list of validation errors.
     """
+    v = Validation()
     if not activity:
-        raise InvalidActivity("empty activity is no activity")
+        v.add_error("activity", "empty activity is no activity")
+        return v.errors()
 
     # when the activity is just a ref, there is little to validate
     ref = activity.get("ref")
     if ref is not None:
         if not isinstance(ref, str) or ref == '':
-            raise InvalidActivity(
-                "reference to activity must be non-empty strings")
-        return
+            v.add_error(
+                "ref", "reference to activity must be non-empty strings")
+        return v.errors()
 
     activity_type = activity.get("type")
     if not activity_type:
-        raise InvalidActivity("an activity must have a type")
+        v.add_error("type", "an activity must have a type")
 
     if activity_type not in ("probe", "action"):
-        raise InvalidActivity(
-            "'{t}' is not a supported activity type".format(t=activity_type))
+        msg = "'{t}' is not a supported activity type".format(t=activity_type)
+        v.add_error("type", msg, value=activity_type)
 
     if not activity.get("name"):
-        raise InvalidActivity("an activity must have a name")
+        v.add_error("name", "an activity must have a name")
 
     provider = activity.get("provider")
     if not provider:
-        raise InvalidActivity("an activity requires a provider")
+        v.add_error("provider", "an activity requires a provider")
+        provider_type = None
+    else:
+        provider_type = provider.get("type")
+        if not provider_type:
+            v.add_error("type", "a provider must have a type")
 
-    provider_type = provider.get("type")
-    if not provider_type:
-        raise InvalidActivity("a provider must have a type")
-
-    if provider_type not in ("python", "process", "http"):
-        raise InvalidActivity(
-            "unknown provider type '{type}'".format(type=provider_type))
-
-    if not activity.get("name"):
-        raise InvalidActivity("activity must have a name (cannot be empty)")
+        if provider_type not in ("python", "process", "http"):
+            msg = "unknown provider type '{type}'".format(type=provider_type)
+            v.add_error("type", msg, value=provider_type)
 
     timeout = activity.get("timeout")
     if timeout is not None:
         if not isinstance(timeout, numbers.Number):
-            raise InvalidActivity("activity timeout must be a number")
+            v.add_error(
+                "timeout", "activity timeout must be a number", value=timeout)
 
     pauses = activity.get("pauses")
     if pauses is not None:
         before = pauses.get("before")
         if before is not None and not isinstance(before, numbers.Number):
-            raise InvalidActivity("activity before pause must be a number")
+            v.add_error(
+                "before", "activity before pause must be a number",
+                value=before
+            )
         after = pauses.get("after")
         if after is not None and not isinstance(after, numbers.Number):
-            raise InvalidActivity("activity after pause must be a number")
+            v.add_error(
+                "after", "activity after pause must be a number",
+                value=after
+            )
 
     if "background" in activity:
         if not isinstance(activity["background"], bool):
-            raise InvalidActivity("activity background must be a boolean")
+            v.add_error(
+                "background", "activity background must be a boolean",
+                value=activity["background"])
 
     if provider_type == "python":
-        validate_python_activity(activity)
+        v.extend_errors(validate_python_activity(activity))
     elif provider_type == "process":
-        validate_process_activity(activity)
+        v.extend_errors(validate_process_activity(activity))
     elif provider_type == "http":
-        validate_http_activity(activity)
+        v.extend_errors(validate_http_activity(activity))
+
+    return v.errors()
 
 
 def run_activities(experiment: Experiment, configuration: Configuration,

chaoslib/control/__init__.py

@@ -7,11 +7,13 @@
 
 from chaoslib.control.python import apply_python_control, cleanup_control, \
     initialize_control, validate_python_control, import_control
-from chaoslib.exceptions import InterruptExecution, InvalidControl
+from chaoslib.exceptions import InterruptExecution, InvalidControl, \
+    ChaosException
 from chaoslib.settings import get_loaded_settings
 from chaoslib.types import Settings
 from chaoslib.types import Activity, Configuration, Control as ControlType, \
-    Experiment, Hypothesis, Journal, Run, Secrets
+    Experiment, Hypothesis, Journal, Run, Secrets, ValidationError
+from chaoslib.validation import Validation
 
 
 __all__ = ["controls", "initialize_controls", "cleanup_controls",
@@ -85,12 +87,12 @@ def cleanup_controls(experiment: Experiment):
             cleanup_control(control)
 
 
-def validate_controls(experiment: Experiment):
+def validate_controls(experiment: Experiment) -> List[ValidationError]:
     """
     Validate that all declared controls respect the specification.
-
-    Raises :exc:`chaoslib.exceptions.InvalidControl` when they are not valid.
     """
+    v = Validation()
+
     controls = get_controls(experiment)
     references = [
         c["name"] for c in get_controls(experiment)
@@ -99,26 +101,33 @@ def validate_controls(experiment: Experiment):
     for c in controls:
         if "ref" in c:
             if c["ref"] not in references:
-                raise InvalidControl(
-                    "Control reference '{}' declaration cannot be found")
+                msg = "Control reference '{}' declaration cannot be found".\
+                    format(c["ref"])
+                v.add_error("ref", msg, value=c["ref"])
 
         if "name" not in c:
-            raise InvalidControl("A control must have a `name` property")
+            v.add_error("name", "A control must have a `name` property")
 
-        name = c["name"]
+        name = c.get("name", '')
         if "provider" not in c:
-            raise InvalidControl(
+            v.add_error(
+                "provider",
                 "Control '{}' must have a `provider` property".format(name))
 
         scope = c.get("scope")
         if scope and scope not in ("before", "after"):
-            raise InvalidControl(
+            v.add_error(
+                "scope",
                 "Control '{}' scope property must be 'before' or "
-                "'after' only".format(name))
+                "'after' only".format(name),
+                value=scope
+            )
 
         provider_type = c.get("provider", {}).get("type")
         if provider_type == "python":
-            validate_python_control(c)
+            v.extend_errors(validate_python_control(c))
+
+    return v.errors()
 
 
 def initialize_global_controls(experiment: Experiment,

chaoslib/control/python.py

@@ -7,9 +7,10 @@
 from logzero import logger
 
 from chaoslib import substitute
-from chaoslib.exceptions import InvalidActivity
+from chaoslib.exceptions import InvalidActivity, ChaosException
 from chaoslib.types import Activity, Configuration, Control, Experiment, \
-    Journal, Run, Secrets, Settings
+    Journal, Run, Secrets, Settings, ValidationError
+from chaoslib.validation import Validation
 
 
 __all__ = ["apply_python_control", "cleanup_control", "initialize_control",
@@ -83,16 +84,22 @@ def cleanup_control(control: Control):
     func()
 
 
-def validate_python_control(control: Control):
+def validate_python_control(control: Control) -> List[ValidationError]:
     """
     Verify that a control block matches the specification
     """
+    v = Validation()
+
     name = control["name"]
     provider = control["provider"]
     mod_name = provider.get("module")
     if not mod_name:
-        raise InvalidActivity(
-            "Control '{}' must have a module path".format(name))
+        v.add_error(
+            "module",
+            "Control '{}' must have a module path".format(name)
+        )
+        # can not continue any longer - must exit this function
+        return v.errors()
 
     try:
         importlib.import_module(mod_name)

chaoslib/exceptions.py

@@ -3,7 +3,7 @@
 __all__ = ["ChaosException", "InvalidExperiment", "InvalidActivity",
            "ActivityFailed", "DiscoveryFailed", "InvalidSource",
            "InterruptExecution", "ControlPythonFunctionLoadingError",
-           "InvalidControl"]
+           "InvalidControl", "ValidationError"]
 
 
 class ChaosException(Exception):
@@ -44,3 +44,7 @@ class InterruptExecution(ChaosException):
 
 class InvalidControl(ChaosException):
     pass
+
+
+class ValidationError(ChaosException):
+    pass

chaoslib/experiment.py

@@ -15,7 +15,8 @@
     cleanup_global_controls
 from chaoslib.deprecation import warn_about_deprecated_features
 from chaoslib.exceptions import ActivityFailed, ChaosException, \
-    InterruptExecution, InvalidActivity, InvalidExperiment
+    InterruptExecution, InvalidActivity, InvalidExperiment, \
+    ValidationError as ValidationErrorException
 from chaoslib.extension import validate_extensions
 from chaoslib.configuration import load_configuration
 from chaoslib.hypothesis import ensure_hypothesis_is_valid, \
@@ -25,14 +26,17 @@
 from chaoslib.secret import load_secrets
 from chaoslib.settings import get_loaded_settings
 from chaoslib.types import Configuration, Experiment, Journal, Run, Secrets, \
-    Settings
+    Settings, ValidationError
+from chaoslib.validation import Validation
 
 initialize_global_controls
 __all__ = ["ensure_experiment_is_valid", "run_experiment", "load_experiment"]
 
 
 @with_cache
-def ensure_experiment_is_valid(experiment: Experiment):
+def ensure_experiment_is_valid(experiment: Experiment,
+                               no_raise: bool = False
+                               ) -> List[ValidationError]:
     """
     A chaos experiment consists of a method made of activities to carry
     sequentially.
@@ -48,54 +52,77 @@ def ensure_experiment_is_valid(experiment: Experiment):
     another set of of  ̀close` probes to sense the state of the system
     post-action.
 
-    This function raises :exc:`InvalidExperiment`, :exc:`InvalidProbe` or
-    :exc:`InvalidAction` depending on where it fails.
+    This function raises an :exc:`InvalidExperiment` error
+    if the experiment is not valid.
+    If multiple validation errors are found, the errors are listed
+    as part of the exception message
+
+    If `no_raise` is True, the function will not raise any
+    exception but rather return the list of validation errors
     """
     logger.info("Validating the experiment's syntax")
 
+    full_validation_msg = "Experiment is not valid, " \
+                          "please fix the following errors. " \
+                          "\n{}"
+    v = Validation()
+
     if not experiment:
-        raise InvalidExperiment("an empty experiment is not an experiment")
+        v.add_error("$", "an empty experiment is not an experiment")
+        # empty experiment, cannot continue validation any further
+        if no_raise:
+            return v.errors()
+        raise InvalidExperiment(full_validation_msg.format(str(v)))
 
     if not experiment.get("title"):
-        raise InvalidExperiment("experiment requires a title")
+        v.add_error("title", "experiment requires a title")
 
     if not experiment.get("description"):
-        raise InvalidExperiment("experiment requires a description")
+        v.add_error("description", "experiment requires a description")
 
     tags = experiment.get("tags")
     if tags:
         if list(filter(lambda t: t == '' or not isinstance(t, str), tags)):
-            raise InvalidExperiment(
-                "experiment tags must be a non-empty string")
+            v.add_error("tags", "experiment tags must be a non-empty string")
 
-    validate_extensions(experiment)
+    v.extend_errors(validate_extensions(experiment))
 
     config = load_configuration(experiment.get("configuration", {}))
     load_secrets(experiment.get("secrets", {}), config)
 
-    ensure_hypothesis_is_valid(experiment)
+    v.extend_errors(ensure_hypothesis_is_valid(experiment))
 
     method = experiment.get("method")
     if not method:
-        raise InvalidExperiment("an experiment requires a method with "
-                                "at least one activity")
-
-    for activity in method:
-        ensure_activity_is_valid(activity)
-
-        # let's see if a ref is indeed found in the experiment
-        ref = activity.get("ref")
-        if ref and not lookup_activity(ref):
-            raise InvalidActivity("referenced activity '{r}' could not be "
-                                  "found in the experiment".format(r=ref))
+        v.add_error(
+            "method",
+            "an experiment requires a method with at least one activity")
+    else:
+        for activity in method:
+            v.extend_errors(ensure_activity_is_valid(activity))
+
+            # let's see if a ref is indeed found in the experiment
+            ref = activity.get("ref")
+            if ref and not lookup_activity(ref):
+                v.add_error(
+                    "ref",
+                    "referenced activity '{r}' could not be "
+                    "found in the experiment".format(r=ref),
+                    value=ref
+                )
 
     rollbacks = experiment.get("rollbacks", [])
     for activity in rollbacks:
-        ensure_activity_is_valid(activity)
+        v.extend_errors(ensure_activity_is_valid(activity))
 
     warn_about_deprecated_features(experiment)
 
-    validate_controls(experiment)
+    v.extend_errors(validate_controls(experiment))
+
+    if v.has_errors():
+        if no_raise:
+            return v.errors()
+        raise InvalidExperiment(full_validation_msg.format(str(v)))
 
     logger.info("Experiment looks valid")