diff --git a/README.md b/README.md index b6d597de..e8b5730e 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ Planned features: * Handling flaky tests with test-specific retries, timeouts * Integration with unittest.mock (specifics to be ironed out) * Plugin system - +re ## Getting Started Install Ward with `pip install ward`. @@ -92,16 +92,10 @@ def test_database_connection(database): expect(users).contains("Bob") ``` -### The Expect API +### The `expect` API -In the (contrived) `test_capital_cities` test, we want to determine whether -the `get_capitals_from_server` function is behaving as expected, -so we grab the output of the function and pass it to `expect`. From -here, we check that the response is as we expect it to be by chaining -methods. If any of the checks fail, the expect chain short-circuits, -and the remaining checks won't be executed for that test. Methods in -the Expect API are named such that they correspond as closely to standard -Python operators as possible, meaning there's not much to memorise. +Use `expect` to perform tests on objects by chaining together methods. Using `expect` allows Ward +to provide detailed, highly readable output when your tests fail. ```python from ward import expect, fixture @@ -119,6 +113,24 @@ def test_capital_cities(cities): .equals(cities)) ``` +Most methods on `expect` have inverted equivalents, e.g. `not_equals`, `not_satisfies`, etc. + +### Working with mocks + +`expect` works well with `unittest.mock`, by providing methods such as `expect.called`, `expect.called_once_with`, +and more. If a test fails due to the mock not being used as expected, Ward will print specialised output to aid +debugging the problem. + +```python +from ward import expect +from unittest.mock import Mock + +def test_mock_was_called(): + mock = Mock() + mock(1, 2, x=3) + expect(mock).called_once_with(1, 2, x=3) +``` + ### Checking for exceptions The test below will pass, because a `ZeroDivisionError` is raised. If a `ZeroDivisionError` wasn't raised, @@ -187,8 +199,8 @@ considered a failure. Check that a value is close to another value. ```python -expect(1.0).approx(1.01, epsilon=0.2) # pass -expect(1.0).approx(1.01, epsilon=0.001) # fail +expect(1.0).approx(1.01, abs_tol=0.2) # pass +expect(1.0).approx(1.01, abs_tol=0.001) # fail ``` ### Cancelling a run after a specific number of failures diff --git a/tests/test_expect.py b/tests/test_expect.py index 655179af..5745cd18 100644 --- a/tests/test_expect.py +++ b/tests/test_expect.py @@ -1,5 +1,7 @@ -from ward import expect -from ward.expect import Expected, ExpectationFailed +from unittest.mock import Mock, patch, call + +from ward import expect, fixture, raises +from ward.expect import Expected, ExpectationFailed, math def test_equals_success_history_recorded(): @@ -15,15 +17,18 @@ def test_equals_failure_history_recorded(): this, that = "hello", "goodbye" e = expect(this) - try: + with raises(ExpectationFailed): e.equals(that) - except ExpectationFailed: - pass hist = [Expected(this=this, op="equals", that=that, op_args=(), op_kwargs={}, success=False)] expect(e.history).equals(hist) +def test_equals_failure_ExpectationFailed_raised(): + with raises(ExpectationFailed): + expect(1).equals(2) + + def test_satisfies_success_history_recorded(): this = "olleh" predicate = lambda e: this[::-1] == "hello" @@ -39,35 +44,249 @@ def test_satisfies_failure_history_recorded(): predicate = lambda e: False e = expect(this) - try: + with raises(ExpectationFailed): e.satisfies(predicate) - except ExpectationFailed: - pass hist = [Expected(this=this, op="satisfies", that=predicate, op_args=(), op_kwargs={}, success=False)] expect(e.history).equals(hist) +def test_satisfies_failure_ExpectationFailed_raised(): + with raises(ExpectationFailed): + expect(1).satisfies(lambda x: False) + + +def test_identical_to_succeeds_when_things_are_identical(): + expect(ZeroDivisionError).identical_to(ZeroDivisionError) + + +def test_identical_to_fails_when_things_are_not_identical(): + with raises(ExpectationFailed): + expect(ZeroDivisionError).identical_to(AttributeError) + + def test_approx_success_history_recorded(): this, that, eps = 1.0, 1.01, 0.5 - e = expect(this).approx(that, epsilon=eps) + e = expect(this).approx(that, abs_tol=eps) - hist = [Expected(this=this, op="approx", that=that, op_args=(), op_kwargs={"epsilon": eps}, success=True)] + hist = [ + Expected( + this=this, op="approx", that=that, op_args=(), op_kwargs={"rel_tol": 1e-09, "abs_tol": 0.5}, success=True + ) + ] expect(e.history).equals(hist) def test_approx_failure_history_recorded(): this, that, eps = 1.0, 1.01, 0.001 - # This will raise an ExpectationFailed, which would fail this test unless we catch it. e = expect(this) - try: + with raises(ExpectationFailed): e.approx(that, eps) - except ExpectationFailed: - pass - hist = [Expected(this=this, op="approx", that=that, op_args=(eps,), op_kwargs={}, success=False)] + hist = [ + Expected( + this=this, op="approx", that=that, op_args=(), op_kwargs={"rel_tol": 0.001, "abs_tol": 0.0}, success=False + ) + ] expect(e.history).equals(hist) + + +@fixture +def isclose(): + with patch.object(math, "isclose", autospec=True) as m: + yield m + + +def test_approx_delegates_to_math_isclose_correctly(isclose): + this, that = 1.0, 1.1 + abs_tol, rel_tol = 0.1, 0.2 + + expect(this).approx(that, abs_tol=abs_tol, rel_tol=rel_tol) + + expect(isclose).called_once_with(this, that, abs_tol=abs_tol, rel_tol=rel_tol) + + +def test_not_approx_delegeates_to_isclose_correctly(isclose): + this, that = 1.0, 1.2 + abs_tol = 0.01 + + with raises(ExpectationFailed): + expect(this).not_approx(that, abs_tol=abs_tol) + + expect(isclose).called_once_with(this, that, abs_tol=abs_tol, rel_tol=1e-9) + + +def test_not_equals_success_history_recorded(): + this, that = 1, 2 + + e = expect(this).not_equals(that) + + hist = [Expected(this, op="not_equals", that=that, op_args=(), op_kwargs={}, success=True)] + expect(e.history).equals(hist) + + +def test_not_equals_failure_history_recorded(): + this, that = 1, 1 + + e = expect(this) + with raises(ExpectationFailed): + e.not_equals(that) + + hist = [Expected(this, op="not_equals", that=that, op_args=(), op_kwargs={}, success=False)] + expect(e.history).equals(hist) + + +@fixture +def mock(): + return Mock() + + +def test_mock_called_when_mock_was_called(mock): + mock() + e = expect(mock).called() + + hist = [Expected(mock, op="called", that=None, op_args=(), op_kwargs={}, success=True)] + expect(e.history).equals(hist) + + +def test_mock_called_when_mock_wasnt_called(mock): + e = expect(mock) + with raises(ExpectationFailed): + e.called() + + hist = [Expected(mock, op="called", that=None, op_args=(), op_kwargs={}, success=False)] + expect(e.history).equals(hist) + + +def test_mock_not_called_success(mock): + e = expect(mock).not_called() + + hist = [Expected(mock, op="not_called", that=None, op_args=(), op_kwargs={}, success=True)] + expect(e.history).equals(hist) + + +def test_mock_called_once_with_success(mock): + args = (1, 2, 3) + kwargs = {"hello": "world"} + mock(*args, **kwargs) + + e = expect(mock).called_once_with(*args, **kwargs) + + hist = [Expected(mock, op="called_once_with", that=None, op_args=args, op_kwargs=kwargs, success=True)] + expect(e.history).equals(hist) + + +def test_mock_called_once_with_failure_missing_arg(mock): + args = (1, 2, 3) + kwargs = {"hello": "world"} + mock(1, 2, **kwargs) # 3 is missing intentionally + + e = expect(mock) + with raises(ExpectationFailed): + e.called_once_with(*args, **kwargs) + + hist = [Expected(mock, op="called_once_with", that=None, op_args=args, op_kwargs=kwargs, success=False)] + expect(e.history).equals(hist) + + +def test_mock_called_once_with_failure_missing_kwarg(mock): + args = (1, 2, 3) + kwargs = {"hello": "world"} + mock(*args, wrong="thing") + + e = expect(mock) + with raises(ExpectationFailed): + e.called_once_with(*args, **kwargs) + + hist = [Expected(mock, op="called_once_with", that=None, op_args=args, op_kwargs=kwargs, success=False)] + expect(e.history).equals(hist) + + +def test_mock_called_once_with_fails_when_multiple_correct_calls(mock): + args = (1, 2, 3) + kwargs = {"hello": "world"} + + mock(*args, **kwargs) + mock(*args, **kwargs) + + e = expect(mock) + with raises(ExpectationFailed): + e.called_once_with(*args, **kwargs) + + hist = [Expected(mock, op="called_once_with", that=None, op_args=args, op_kwargs=kwargs, success=False)] + expect(e.history).equals(hist) + + +def test_mock_called_once_with_fails_when_multiple_calls_but_one_correct(mock): + args = (1, 2, 3) + kwargs = {"hello": "world"} + + mock(1) + mock(*args, **kwargs) + mock(2) + + e = expect(mock) + with raises(ExpectationFailed): + e.called_once_with(*args, **kwargs) + + hist = [Expected(mock, op="called_once_with", that=None, op_args=args, op_kwargs=kwargs, success=False)] + expect(e.history).equals(hist) + + +def test_called_with_succeeds_when_expected_call_is_last(mock): + mock(1) + mock(2) + + e = expect(mock).called_with(2) + expect(e.history[0].success).equals(True) + + +def test_called_with_fails_when_expected_call_is_made_but_not_last(mock): + mock(2) + mock(1) + e = expect(mock) + with raises(ExpectationFailed): + e.called_with(2) + expect(e.history[0].success).equals(False) + + +def test_has_calls_succeeds_when_all_calls_were_made(mock): + mock(1, 2) + mock(key="value") + + e = expect(mock).has_calls([call(1, 2), call(key="value")]) + expect(e.history[0].success).equals(True) + + +def test_has_calls_fails_when_not_all_calls_were_made(mock): + mock(1, 2) + + e = expect(mock) + with raises(ExpectationFailed): + e.has_calls([call(1, 2), call(key="value")]) + expect(e.history[0].success).equals(False) + + +def test_has_calls_fails_when_calls_were_made_in_wrong_order(mock): + mock(1, 2) + mock(key="value") + + e = expect(mock) + with raises(ExpectationFailed): + e.has_calls([call(key="value"), call(1, 2)]) + + +def test_has_calls_succeeds_when_all_calls_were_made_any_order(mock): + mock(1, 2) + mock(key="value") + + e = expect(mock).has_calls( + [call(key="value"), call(1, 2)], + any_order=True, + ) + + expect(e.history[0].success).equals(True) diff --git a/tests/test_suite.py b/tests/test_suite.py index 271dcd2f..a78152b6 100644 --- a/tests/test_suite.py +++ b/tests/test_suite.py @@ -80,7 +80,7 @@ def test_i_fail(): expected_result = TestResult(test=test, outcome=TestOutcome.FAIL, error=mock.ANY, message="") expect(result).equals(expected_result) - expect(result.error).is_instance_of(AssertionError) + expect(result.error).instance_of(AssertionError) def test_generate_test_runs__yields_skipped_test_result_on_test_with_skip_marker( @@ -127,3 +127,4 @@ def my_test(fix_a, fix_b): list(suite.generate_test_runs()) expect(events).equals([1, 2, 3]) + diff --git a/ward/expect.py b/ward/expect.py index 370de5ab..e7e15d75 100644 --- a/ward/expect.py +++ b/ward/expect.py @@ -1,7 +1,9 @@ import functools +import inspect import math from dataclasses import dataclass -from typing import Type, Any, List, Callable, Dict, Tuple +from typing import Type, Any, List, Callable, Dict, Tuple, Optional +from unittest.mock import _Call class raises: @@ -21,7 +23,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): class Expected: this: Any op: str - that: Any + that: Optional[Any] op_args: Tuple op_kwargs: Dict success: bool = True @@ -56,38 +58,130 @@ def __init__(self, this: Any): self.this = this self.history: List[Expected] = [] - @record_and_handle_outcome - def equals(self, that: Any): - return self.this == that + def equals(self, expected: Any): + return self._handle_expect(self.this == expected, that=expected) - @record_and_handle_outcome - def is_less_than(self, that: Any): - return self.this < that + def not_equals(self, that: Any): + return self._handle_expect(self.this != that, that=that) - @record_and_handle_outcome - def is_greater_than(self, that: Any): - return self.this > that + def less_than(self, that: Any): + return self._handle_expect(self.this < that, that=that) + + def not_less_than(self, that: Any): + return self._handle_expect(not self.this < that, that=that) + + def less_than_or_equals(self, that: Any): + return self._handle_expect(self.this <= that, that=that) + + def not_less_than_or_equals(self, that: Any): + return self._handle_expect(not self.this <= that, that=that) + + def greater_than(self, that: Any): + return self._handle_expect(self.this > that, that=that) + + def not_greater_than(self, that: Any): + return self._handle_expect(not self.this > that, that=that) + + def greater_than_or_equals(self, that: Any): + return self._handle_expect(self.this >= that, that=that) + + def not_greater_than_or_equals(self, that: Any): + return self._handle_expect(not self.this >= that, that=that) - @record_and_handle_outcome def contains(self, that: Any): - return that in self.this + return self._handle_expect(that in self.this, that=that) + + def not_contains(self, that: Any): + return self._handle_expect(that not in self.this, that=that) - @record_and_handle_outcome def has_length(self, length: int): - return len(self.this) == length + return self._handle_expect(len(self.this) == length, that=length) - @record_and_handle_outcome - def is_instance_of(self, type: Type): - return isinstance(self.this, type) + def not_has_length(self, length: int): + return self._handle_expect(len(self.this) != length, that=length) - @record_and_handle_outcome - def satisfies(self, predicate: Callable[["expect"], bool]): - return predicate(self.this) + def instance_of(self, type: Type): + return self._handle_expect(isinstance(self.this, type), that=type) - @record_and_handle_outcome - def is_(self, that: Any): - return self.this is that + def not_instance_of(self, type: Type): + return self._handle_expect(not isinstance(self.this, type), that=type) - @record_and_handle_outcome - def approx(self, that: Any, epsilon: float): - return math.isclose(self.this, that, abs_tol=epsilon) + def satisfies(self, predicate: Callable[["expect"], bool]): + return self._handle_expect(predicate(self.this), that=predicate) + + def not_satisfies(self, predicate: Callable[["expect"], bool]): + return self._handle_expect(not predicate(self.this), that=predicate) + + def identical_to(self, that: Any): + return self._handle_expect(self.this is that, that=that) + + def not_identical_to(self, that: Any): + return self._handle_expect(self.this is not that, that=that) + + def approx(self, that: Any, rel_tol: float = 1e-9, abs_tol: float = 0.0): + return self._handle_expect( + math.isclose(self.this, that, abs_tol=abs_tol, rel_tol=rel_tol), that=that, rel_tol=rel_tol, abs_tol=abs_tol + ) + + def not_approx(self, that: Any, rel_tol: float = 1e-9, abs_tol: float = 0.0): + return self._handle_expect( + not math.isclose(self.this, that, abs_tol=abs_tol, rel_tol=rel_tol), + that=that, + rel_tol=rel_tol, + abs_tol=abs_tol, + ) + + def called(self): + return self._handle_expect(self.this.called) + + def not_called(self): + return self._handle_expect(not self.this.called) + + def called_once_with(self, *args, **kwargs): + try: + self.this.assert_called_once_with(*args, **kwargs) + passed = True + except AssertionError: + passed = False + return self._handle_expect(passed, *args, **kwargs) + + def called_with(self, *args, **kwargs): + try: + self.this.assert_called_with(*args, **kwargs) + passed = True + except AssertionError: + passed = False + return self._handle_expect(passed, *args, **kwargs) + + def has_calls(self, calls: List[_Call], any_order: bool = False): + try: + self.this.assert_has_calls(calls, any_order=any_order) + passed = True + except AssertionError: + passed = False + return self._handle_expect(passed, calls=calls, any_order=any_order) + + def _store_in_history( + self, result: bool, called_with_args: Tuple[Any], called_with_kwargs: Dict[str, Any], that=None + ) -> bool: + self.history.append( + Expected( + this=self.this, + op=inspect.stack()[2].function, # :) + that=that, + success=result, + op_args=called_with_args, + op_kwargs=called_with_kwargs, + ) + ) + return result + + def _fail_if_false(self, val: bool) -> bool: + if val: + return True + raise ExpectationFailed("expectation failed", self.history) + + def _handle_expect(self, result: bool, *args, that: Any = None, **kwargs) -> "expect": + self._store_in_history(result, that=that, called_with_args=args, called_with_kwargs=kwargs) + self._fail_if_false(result) + return self diff --git a/ward/suite.py b/ward/suite.py index 1a0304af..c0395c62 100644 --- a/ward/suite.py +++ b/ward/suite.py @@ -33,7 +33,7 @@ def generate_test_runs(self) -> Generator[TestResult, None, None]: resolved_fixtures = test.resolve_args(self.fixture_registry) except FixtureExecutionError as e: yield TestResult( - test, TestOutcome.FAIL, e, captured_stdout=sout.getvalue(), captured_stderr=serr.getvalue() + test, TestOutcome.FAIL, e, captucared_stdout=sout.getvalue(), captured_stderr=serr.getvalue() ) sout.close() serr.close() @@ -61,6 +61,8 @@ def generate_test_runs(self) -> Generator[TestResult, None, None]: test, TestOutcome.FAIL, e, captured_stdout=sout.getvalue(), captured_stderr=serr.getvalue() ) finally: + # TODO: Don't just cleanup top-level dependencies, since there may + # be generator fixtures elsewhere in the tree requiring cleanup for fixture in resolved_fixtures.values(): if fixture.is_generator_fixture: with suppress(RuntimeError, StopIteration): diff --git a/ward/terminal.py b/ward/terminal.py index 416aa6da..1848fd90 100644 --- a/ward/terminal.py +++ b/ward/terminal.py @@ -2,16 +2,16 @@ import sys import traceback from dataclasses import dataclass -from typing import Generator, List, Optional, Dict +from typing import Dict, Generator, List, Optional from colorama import Fore, Style -from termcolor import colored +from termcolor import colored, cprint from ward.diff import make_diff -from ward.expect import ExpectationFailed +from ward.expect import ExpectationFailed, Expected from ward.suite import Suite from ward.test_result import TestOutcome, TestResult -from ward.util import get_exit_code, ExitCode +from ward.util import ExitCode, get_exit_code def truncate(s: str, num_chars: int) -> str: @@ -122,45 +122,85 @@ def output_single_test_result(self, test_result: TestResult): print(colored(padded_outcome, color="grey", on_color=bg), mod_name + test_result.test.name) def output_why_test_failed_header(self, test_result: TestResult): - print(colored(" Failure", color="red"), "in", colored(test_result.test.qualified_name, attrs=["bold"])) + params_list = ", ".join(lightblack(str(v)) for v in test_result.test.deps().keys()) + if test_result.test.has_deps(): + test_name_suffix = f"({params_list})" + else: + test_name_suffix = "" + print( + colored(" Failure", color="red"), + "in", + colored(test_result.test.qualified_name, attrs=["bold"]), + test_name_suffix, + "\n", + ) def output_why_test_failed(self, test_result: TestResult): - truncation_chars = self.terminal_size.width - 24 err = test_result.error if isinstance(err, ExpectationFailed): - print(f"\n Given {truncate(repr(err.history[0].this), num_chars=truncation_chars)}\n") + print(f" Given {truncate(repr(err.history[0].this), num_chars=self.terminal_size.width - 24)}\n") for expect in err.history: - if expect.success: - result_marker = f"[ {Fore.GREEN}✓{Style.RESET_ALL} ]{Fore.GREEN}" - else: - result_marker = f"[ {Fore.RED}✗{Style.RESET_ALL} ]{Fore.RED}" - - if expect.op == "satisfies" and hasattr(expect.that, "__name__"): - expect_that = truncate(expect.that.__name__, num_chars=truncation_chars) - else: - expect_that = truncate(repr(expect.that), num_chars=truncation_chars) - print(f" {result_marker} it {expect.op} {expect_that}{Style.RESET_ALL}") - - if err.history and err.history[-1].op == "equals": - expect = err.history[-1] - print( - f"\n Showing diff of {colored('expected value', color='green')}" - f" vs {colored('actual value', color='red')}:\n" - ) - - diff = make_diff(expect.that, expect.this, width=truncation_chars) - print(diff) + self.print_expect_chain_item(expect) + + last_check = err.history[-1].op # the check that failed + if last_check == "equals": + self.print_failure_equals(err) else: - trace = getattr(err, "__traceback__", "") - if trace: - trc = traceback.format_exception(None, err, trace) - print("".join(trc)) - else: - print(str(err)) + self.print_traceback(err) print(Style.RESET_ALL) + def print_failure_equals(self, err): + expect = err.history[-1] + print( + f"\n Showing diff of {colored('expected value', color='green')}" + f" vs {colored('actual value', color='red')}:\n" + ) + diff = make_diff(expect.that, expect.this, width=self.terminal_size.width - 24) + print(diff) + + def print_traceback(self, err): + trace = getattr(err, "__traceback__", "") + if trace: + trc = traceback.format_exception(None, err, trace) + if err.__cause__: + cause = err.__cause__.__class__.__name__ + else: + cause = None + for line in trc: + sublines = line.split("\n") + for subline in sublines: + content = " " * 4 + subline + if subline.lstrip().startswith("File \""): + cprint(content, color="yellow") + elif subline.lstrip().startswith("Traceback"): + cprint(content, color="blue") + elif (subline.lstrip().startswith(err.__class__.__name__) or + (cause and subline.lstrip().startswith(cause))): + cprint(content, color="blue") + else: + print(content) + else: + print(str(err)) + + def print_expect_chain_item(self, expect: Expected): + checkbox = self.result_checkbox(expect) + that_width = self.terminal_size.width - 32 + if expect.op == "satisfies" and hasattr(expect.that, "__name__"): + expect_that = truncate(expect.that.__name__, num_chars=that_width) + else: + that = repr(expect.that) if expect.that else "" + expect_that = truncate(that, num_chars=that_width) + print(f" {checkbox} it {expect.op} {expect_that}{Style.RESET_ALL}") + + def result_checkbox(self, expect): + if expect.success: + result_marker = f"[ {Fore.GREEN}✓{Style.RESET_ALL} ]{Fore.GREEN}" + else: + result_marker = f"[ {Fore.RED}✗{Style.RESET_ALL} ]{Fore.RED}" + return result_marker + def output_test_result_summary(self, test_results: List[TestResult], time_taken: float): outcome_counts = self._get_outcome_counts(test_results) chart = self.generate_chart( @@ -193,12 +233,13 @@ def output_captured_stderr(self, test_result: TestResult): print(f" Captured {stderr} during test run:\n") for line in captured_stderr_lines: print(" " + line) + print() def output_captured_stdout(self, test_result: TestResult): if test_result.captured_stdout: stdout = colored("standard output", color="blue") captured_stdout_lines = test_result.captured_stdout.split("\n") - print(f"\n Captured {stdout} during test run:\n") + print(f" Captured {stdout} during test run:\n") for line in captured_stdout_lines: print(" " + line)