Add finally block (rebased)

tavern/_core/exceptions.py

@@ -1,6 +1,15 @@
+from typing import TYPE_CHECKING, Dict, Optional
+
+if TYPE_CHECKING:
+    from tavern._core.pytest.config import TestConfig
+
+
 class TavernException(Exception):
     """Base exception"""
 
+    stage: Optional[Dict]
+    test_block_config: Optional["TestConfig"]
+
 
 class BadSchemaError(TavernException):
     """Schema mismatch"""

tavern/_core/plugins.py

@@ -4,11 +4,14 @@
 significantly if/when a proper plugin system is implemented!
 """
 import logging
+from typing import Any, Dict
 
 import stevedore
 
 from tavern._core import exceptions
 from tavern._core.dict_util import format_keys
+from tavern._core.pytest.config import TestConfig
+from tavern.request import BaseRequest
 
 logger = logging.getLogger(__name__)
 
@@ -161,15 +164,17 @@ def get_extra_sessions(test_spec, test_block_config):
     return sessions
 
 
-def get_request_type(stage, test_block_config, sessions):
+def get_request_type(
+    stage: dict, test_block_config: TestConfig, sessions: Dict[str, PluginHelperBase]
+) -> BaseRequest:
     """Get the request object for this stage
 
     there can only be one
 
     Args:
-        stage (dict): spec for this stage
-        test_block_config (dict): variables for this test run
-        sessions (dict): all available sessions
+        stage: spec for this stage
+        test_block_config: variables for this test run
+        sessions: all available sessions
 
     Returns:
         BaseRequest: request object with a run() method
@@ -217,12 +222,12 @@ class ResponseVerifier(dict):
     plugin_name: str
 
 
-def _foreach_response(stage, test_block_config, action):
+def _foreach_response(stage: dict, test_block_config: TestConfig, action):
     """Do something for each response
 
     Args:
-        stage (dict): Stage of test
-        test_block_config (dict): Config for test
+        stage : Stage of test
+        test_block_config : Config for test
         action ((p: {plugin, name}, response_block: dict) -> Any): function that takes (plugin, response block)
 
     Returns:
@@ -241,7 +246,9 @@ def _foreach_response(stage, test_block_config, action):
     return retvals
 
 
-def get_expected(stage, test_block_config, sessions):
+def get_expected(
+    stage: dict, test_block_config: TestConfig, sessions: Dict[str, PluginHelperBase]
+):
     """Get expected responses for each type of request
 
     Though only 1 request can be made, it can cause multiple responses.
@@ -251,9 +258,9 @@ def get_expected(stage, test_block_config, sessions):
     BEFORE running the request.
 
     Args:
-        stage (dict): test stage
-        test_block_config (dict): available configuration for this test
-        sessions (dict): all available sessions
+        stage: test stage
+        test_block_config: available configuration for this test
+        sessions : all available sessions
 
     Returns:
         dict: mapping of request type: expected response dict
@@ -273,13 +280,18 @@ def action(p, response_block):
     return _foreach_response(stage, test_block_config, action)
 
 
-def get_verifiers(stage, test_block_config, sessions, expected):
+def get_verifiers(
+    stage: dict,
+    test_block_config: TestConfig,
+    sessions: Dict[str, PluginHelperBase],
+    expected: Dict[str, Any],
+):
     """Get one or more response validators for this stage
 
     Args:
-        stage (dict): spec for this stage
-        test_block_config (dict): variables for this test run
-        sessions (dict): all available sessions
+        stage: spec for this stage
+        test_block_config: variables for this test run
+        sessions: all available sessions
         expected (dict): expected responses for this stage
 
     Returns:

tavern/_core/run.py

@@ -1,21 +1,25 @@
 from contextlib import ExitStack
 import copy
 from copy import deepcopy
+import dataclasses
 from distutils.util import strtobool  # pylint: disable=deprecated-module
 import functools
 import logging
+from typing import Dict
 
 from tavern._core import exceptions
 from tavern._core.plugins import (
+    PluginHelperBase,
     get_expected,
     get_extra_sessions,
     get_request_type,
     get_verifiers,
 )
-from tavern._core.strict_util import StrictLevel
+from tavern._core.strict_util import StrictLevel, StrictSetting
 
 from .dict_util import format_keys, get_tavern_box
 from .pytest import call_hook
+from .pytest.config import TestConfig
 from .report import attach_stage_content, wrap_step
 from .testhelpers import delay, retry
 
@@ -122,7 +126,7 @@ def run_test(in_file, test_spec, global_cfg):
     # Initialise test config for this test with the global configuration before
     # starting
     test_block_config = global_cfg.copy()
-    default_global_stricness = global_cfg.strict
+    default_global_strictness = global_cfg.strict
 
     tavern_box = get_tavern_box()
 
@@ -162,41 +166,25 @@ def getonly(stage):
 
         has_only = any(getonly(stage) for stage in test_spec["stages"])
 
-        # Run tests in a path in order
-        for idx, stage in enumerate(test_spec["stages"]):
-            if stage.get("skip"):
-                continue
-            if has_only and not getonly(stage):
-                continue
-
-            test_block_config = test_block_config.with_strictness(
-                default_global_stricness
-            )
-            test_block_config = test_block_config.with_strictness(
-                _calculate_stage_strictness(stage, test_block_config, test_spec)
-            )
-
-            # Wrap run_stage with retry helper
-            run_stage_with_retries = retry(stage, test_block_config)(run_stage)
-
-            partial = functools.partial(
-                run_stage_with_retries, sessions, stage, test_block_config
-            )
+        runner = _TestRunner(
+            default_global_strictness, sessions, test_block_config, test_spec
+        )
 
-            allure_name = "Stage {}: {}".format(
-                idx, format_keys(stage["name"], test_block_config.variables)
-            )
-            step = wrap_step(allure_name, partial)
+        try:
+            # Run tests in a path in order
+            for idx, stage in enumerate(test_spec["stages"]):
+                if stage.get("skip"):
+                    continue
+                if has_only and not getonly(stage):
+                    continue
 
-            try:
-                step()
-            except exceptions.TavernException as e:
-                e.stage = stage
-                e.test_block_config = test_block_config
-                raise
+                runner.wrap_run_stage(idx, stage)
 
-            if getonly(stage):
-                break
+                if getonly(stage):
+                    break
+        finally:
+            for idx, stage in enumerate(test_spec.get("finally", [])):
+                runner.wrap_run_stage(idx, stage)
 
 
 def _calculate_stage_strictness(stage, test_block_config, test_spec):
@@ -259,45 +247,73 @@ def update_stage_options(new_option):
     return new_strict
 
 
-def run_stage(sessions, stage, test_block_config):
-    """Run one stage from the test
+@dataclasses.dataclass(frozen=True)
+class _TestRunner:
+    default_global_strictness: StrictSetting
+    sessions: Dict[str, PluginHelperBase]
+    test_block_config: TestConfig
+    test_spec: Dict
 
-    Args:
-        sessions (dict): Dictionary of relevant 'session' objects used for this test
-        stage (dict): specification of stage to be run
-        test_block_config (TestConfig): available variables for test
-    """
-    stage = copy.deepcopy(stage)
-    name = stage["name"]
+    def wrap_run_stage(self, idx: int, stage):
+        stage_config = self.test_block_config.with_strictness(
+            self.default_global_strictness
+        )
+        stage_config = stage_config.with_strictness(
+            _calculate_stage_strictness(stage, stage_config, self.test_spec)
+        )
+        # Wrap run_stage with retry helper
+        run_stage_with_retries = retry(stage, stage_config)(self.run_stage)
+        partial = functools.partial(run_stage_with_retries, stage, stage_config)
+        allure_name = "Stage {}: {}".format(
+            idx, format_keys(stage["name"], stage_config.variables)
+        )
+        step = wrap_step(allure_name, partial)
 
-    attach_stage_content(stage)
+        try:
+            step()
+        except exceptions.TavernException as e:
+            e.stage = stage
+            e.test_block_config = stage_config
+            raise
 
-    r = get_request_type(stage, test_block_config, sessions)
+    def run_stage(self, stage: dict, stage_config: TestConfig):
+        """Run one stage from the test
 
-    tavern_box = test_block_config.variables["tavern"]
-    tavern_box.update(request_vars=r.request_vars)
+        Args:
+            stage: specification of stage to be run
+            stage_config: available variables for test
+        """
+        stage = copy.deepcopy(stage)
+        name = stage["name"]
 
-    expected = get_expected(stage, test_block_config, sessions)
+        attach_stage_content(stage)
 
-    delay(stage, "before", test_block_config.variables)
+        r = get_request_type(stage, stage_config, self.sessions)
 
-    logger.info("Running stage : %s", name)
+        tavern_box = stage_config.variables["tavern"]
+        tavern_box.update(request_vars=r.request_vars)
 
-    call_hook(
-        test_block_config,
-        "pytest_tavern_beta_before_every_request",
-        request_args=r.request_vars,
-    )
+        expected = get_expected(stage, stage_config, self.sessions)
+
+        delay(stage, "before", stage_config.variables)
+
+        logger.info("Running stage : %s", name)
+
+        call_hook(
+            stage_config,
+            "pytest_tavern_beta_before_every_request",
+            request_args=r.request_vars,
+        )
 
-    verifiers = get_verifiers(stage, test_block_config, sessions, expected)
+        verifiers = get_verifiers(stage, stage_config, self.sessions, expected)
 
-    response = r.run()
+        response = r.run()
 
-    for response_type, response_verifiers in verifiers.items():
-        logger.debug("Running verifiers for %s", response_type)
-        for v in response_verifiers:
-            saved = v.verify(response)
-            test_block_config.variables.update(saved)
+        for response_type, response_verifiers in verifiers.items():
+            logger.debug("Running verifiers for %s", response_type)
+            for v in response_verifiers:
+                saved = v.verify(response)
+                stage_config.variables.update(saved)
 
-    tavern_box.pop("request_vars")
-    delay(stage, "after", test_block_config.variables)
+        tavern_box.pop("request_vars")
+        delay(stage, "after", stage_config.variables)

tavern/_core/schema/tests.jsonschema.yaml

@@ -423,3 +423,10 @@ properties:
       oneOf:
       - $ref: "#/definitions/stage"
       - $ref: "#/definitions/stage_ref"
+
+  finally:
+    type: array
+    description: Stages to run after test finishes
+
+    items:
+      $ref: "#/definitions/stage"

tests/integration/test_control_flow.tavern.yaml

@@ -0,0 +1,53 @@
+---
+
+test_name: Test finally block doing nothing
+
+stages:
+  - name: Simple echo
+    request:
+      url: "{host}/echo"
+      method: POST
+      json:
+        value: "123"
+    response:
+      status_code: 200
+      json:
+        value: "123"
+
+finally:
+  - name: nothing
+    request:
+      url: "{host}/echo"
+      method: POST
+      json:
+        value: "123"
+
+---
+
+test_name: Test finally block fail
+
+_xfail: run
+
+stages:
+  - name: Simple echo
+    request:
+      url: "{host}/echo"
+      method: POST
+      json:
+        value: "123"
+    response:
+      status_code: 200
+      json:
+        value: "123"
+
+finally:
+  - name: nothing
+    request:
+      url: "{host}/echo"
+      method: DELETE
+      json:
+        value: "123"
+    response:
+      status_code: 200
+      json:
+        value: "123"