diff --git a/.circleci/config.yml b/.circleci/config.yml index 1139fc4..c7dfaca 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -73,6 +73,22 @@ jobs: # This file is automatically generated by ../rebuild-circleci-yaml # !!! WARNING !!! + test-py27-pytest4: + docker: + - image: python:2.7 + steps: + - checkout + - run: + name: install tox + command: pip install tox==3.7.0 + - run: + name: run tests + command: tox -e py27-pytest4 + + # !!! WARNING !!! + # This file is automatically generated by ../rebuild-circleci-yaml + # !!! WARNING !!! + test-py27-unit: docker: - image: python:2.7 @@ -169,6 +185,22 @@ jobs: # This file is automatically generated by ../rebuild-circleci-yaml # !!! WARNING !!! + test-py35-pytest4: + docker: + - image: python:3.5 + steps: + - checkout + - run: + name: install tox + command: pip install tox==3.7.0 + - run: + name: run tests + command: tox -e py35-pytest4 + + # !!! WARNING !!! + # This file is automatically generated by ../rebuild-circleci-yaml + # !!! WARNING !!! + test-py35-unit: docker: - image: python:3.5 @@ -265,6 +297,22 @@ jobs: # This file is automatically generated by ../rebuild-circleci-yaml # !!! WARNING !!! + test-py36-pytest4: + docker: + - image: python:3.6 + steps: + - checkout + - run: + name: install tox + command: pip install tox==3.7.0 + - run: + name: run tests + command: tox -e py36-pytest4 + + # !!! WARNING !!! + # This file is automatically generated by ../rebuild-circleci-yaml + # !!! WARNING !!! + test-py36-unit: docker: - image: python:3.6 @@ -361,6 +409,22 @@ jobs: # This file is automatically generated by ../rebuild-circleci-yaml # !!! WARNING !!! + test-pypy-pytest4: + docker: + - image: pypy:2 + steps: + - checkout + - run: + name: install tox + command: pip install tox==3.7.0 + - run: + name: run tests + command: tox -e pypy-pytest4 + + # !!! WARNING !!! + # This file is automatically generated by ../rebuild-circleci-yaml + # !!! WARNING !!! + test-pypy-unit: docker: - image: pypy:2 @@ -441,6 +505,22 @@ jobs: # This file is automatically generated by ../rebuild-circleci-yaml # !!! WARNING !!! + test-py37-pytest4: + docker: + - image: python:3.7 + steps: + - checkout + - run: + name: install tox + command: pip install tox==3.7.0 + - run: + name: run tests + command: tox -e py37-pytest4 + + # !!! WARNING !!! + # This file is automatically generated by ../rebuild-circleci-yaml + # !!! WARNING !!! + test-py37-unit: docker: - image: python:3.7 @@ -521,6 +601,22 @@ jobs: # This file is automatically generated by ../rebuild-circleci-yaml # !!! WARNING !!! + test-py38-pytest4: + docker: + - image: python:3.8 + steps: + - checkout + - run: + name: install tox + command: pip install tox==3.7.0 + - run: + name: run tests + command: tox -e py38-pytest4 + + # !!! WARNING !!! + # This file is automatically generated by ../rebuild-circleci-yaml + # !!! WARNING !!! + test-py38-unit: docker: - image: python:3.8 @@ -555,7 +651,7 @@ jobs: test-py39-nose: docker: - - image: python:3.9-rc + - image: python:3.9 steps: - checkout - run: @@ -571,7 +667,7 @@ jobs: test-py39-nose2: docker: - - image: python:3.9-rc + - image: python:3.9 steps: - checkout - run: @@ -587,7 +683,7 @@ jobs: test-py39-pytest3: docker: - - image: python:3.9-rc + - image: python:3.9 steps: - checkout - run: @@ -601,9 +697,25 @@ jobs: # This file is automatically generated by ../rebuild-circleci-yaml # !!! WARNING !!! + test-py39-pytest4: + docker: + - image: python:3.9 + steps: + - checkout + - run: + name: install tox + command: pip install tox==3.7.0 + - run: + name: run tests + command: tox -e py39-pytest4 + + # !!! WARNING !!! + # This file is automatically generated by ../rebuild-circleci-yaml + # !!! WARNING !!! + test-py39-unit: docker: - - image: python:3.9-rc + - image: python:3.9 steps: - checkout - run: @@ -619,7 +731,7 @@ jobs: test-py39-unit2: docker: - - image: python:3.9-rc + - image: python:3.9 steps: - checkout - run: @@ -637,39 +749,46 @@ workflows: - test-py27-nose2 - test-py27-pytest2 - test-py27-pytest3 + - test-py27-pytest4 - test-py27-unit - test-py27-unit2 - test-py35-nose - test-py35-nose2 - test-py35-pytest2 - test-py35-pytest3 + - test-py35-pytest4 - test-py35-unit - test-py35-unit2 - test-py36-nose - test-py36-nose2 - test-py36-pytest2 - test-py36-pytest3 + - test-py36-pytest4 - test-py36-unit - test-py36-unit2 - test-pypy-nose - test-pypy-nose2 - test-pypy-pytest2 - test-pypy-pytest3 + - test-pypy-pytest4 - test-pypy-unit - test-pypy-unit2 - test-py37-nose - test-py37-nose2 - test-py37-pytest3 + - test-py37-pytest4 - test-py37-unit - test-py37-unit2 - test-py38-nose - test-py38-nose2 - test-py38-pytest3 + - test-py38-pytest4 - test-py38-unit - test-py38-unit2 - test-py39-nose - test-py39-nose2 - test-py39-pytest3 + - test-py39-pytest4 - test-py39-unit - test-py39-unit2 # !!! WARNING !!! diff --git a/README.rst b/README.rst index ff0a117..4ff5318 100644 --- a/README.rst +++ b/README.rst @@ -13,11 +13,9 @@ Parameterized testing with any Python test framework :alt: Circle CI :target: https://circleci.com/gh/wolever/parameterized - -Parameterized testing in Python sucks. - -``parameterized`` fixes that. For everything. Parameterized testing for nose, -parameterized testing for py.test, parameterized testing for unittest. +``parameterized`` provides universal parameterized testing for Python: +parameterized testing for nose, parameterized testing for py.test, +parameterized testing for unittest, parameterized testing for Django. .. code:: python @@ -135,7 +133,7 @@ With unittest (and unittest2):: (note: because unittest does not support test decorators, only tests created with ``@parameterized.expand`` will be executed) -With green:: +With `green`__ :: $ green test_math.py -vvv test_math @@ -165,6 +163,7 @@ With green:: OK (passes=9) +__ https://github.com/CleanCut/green Installation ------------ @@ -241,16 +240,16 @@ __ https://travis-ci.org/wolever/parameterized - yes - yes * - py.test 4 - - no** - - no** - - no** - - no** - - no** - - no** - - no** - - no** - - no** - - no** + - yes + - yes + - yes + - yes + - yes + - yes + - yes + - yes + - yes + - yes * - py.test fixtures - no† - no† @@ -289,8 +288,6 @@ __ https://travis-ci.org/wolever/parameterized \*: py.test 2 does `does not appear to work (#71)`__ under Python 3. Please comment on the related issues if you are affected. -\*\*: py.test 4 is not yet supported (but coming!) in `issue #34`__ - †: py.test fixture support is documented in `issue #81`__ __ https://github.com/wolever/parameterized/issues/71 @@ -579,7 +576,6 @@ which controls the name of the parameterized classes generated by test_concat (test_concat.TestConcatenation_0_say_cheese__) ... ok - Using with Single Parameters ............................ @@ -620,15 +616,42 @@ can be confusing. The ``@mock.patch(...)`` decorator must come *below* the .. code:: python - @mock.patch("os.getpid") class TestOS(object): @parameterized(...) @mock.patch("os.fdopen") @mock.patch("os.umask") - def test_method(self, param1, param2, ..., mock_umask, mock_fdopen, mock_getpid): + def test_method(self, param1, param2, ..., mock_umask, mock_fdopen): ... -Note: the same holds true when using ``@parameterized.expand``. +Note 1: the same holds true when using ``@parameterized.expand``. + +Note 2: ``@mock.patch`` is supported with all runners, including ``pytest``, +*except* when used as a *class decorator* with ``pytest>=4``. + +Parameterized testing with Django +................................. + +``parameterized`` enables parameterized testing with Django with +``@parameterized.expand``:: + + from django.test import TestCase + + class DjangoTestCase(TestCase): + @parameterized.expand([ + ("negative", -1.5, -2.0), + ("integer", 1, 1.0), + ("large fraction", 1.6, 1), + ]) + def test_floor(self, name, input, expected): + assert_equal(math.floor(input), expected) + +Which will yield:: + + $ python manage.py test + ... + test_floor_0_negative (test_math.DjangoTestCase) ... ok + test_floor_1_integer (test_math.DjangoTestCase) ... ok + test_floor_2_large_fraction (test_math.DjangoTestCase) ... ok Migrating from ``nose-parameterized`` to ``parameterized`` @@ -654,7 +677,7 @@ What happened to ``nose-parameterized``? only made sense to change the name! What do you mean when you say "nose is best supported"? - There are small caveates with ``py.test`` and ``unittest``: ``py.test`` + There are small caveats with ``py.test`` and ``unittest``: ``py.test`` does not show the parameter values (ex, it will show ``test_add[0]`` instead of ``test_add[1, 2, 3]``), and ``unittest``/``unittest2`` do not support test generators so ``@parameterized.expand`` must be used. @@ -668,3 +691,26 @@ Why do I get an ``AttributeError: 'function' object has no attribute 'expand'`` You've likely installed the ``parametrized`` (note the missing *e*) package. Use ``parameterized`` (with the *e*) instead and you'll be all set. + +What is the ``no value for arguments`` error when using ``pytest>=4``? + The ``no value for arguments`` error occurs with ``pytest>=4`` when the + parameters for a method do not supply values for all the test function + arguments. + + For example, consider:: + + @parameterized([ + (1, ), + (2, 3), + ]) + def test_foo(a, b): + pass + + In this case, the error will be ``no value for arguments: 'b' with + paramters (1, )``, because the parameter ``(1, )`` does not provide + a value for the argument ``b``. + + Because ``pytest.mark.parametrized`` - which is used to implement + parametrized testing with ``pytest>=4`` - depends fairly heavily on + argument names, this can also come up if other decorators are used (for + example, if ``@mock.patch`` is used as a class decorator). diff --git a/parameterized/parameterized.py b/parameterized/parameterized.py index 969a157..3642923 100644 --- a/parameterized/parameterized.py +++ b/parameterized/parameterized.py @@ -19,9 +19,14 @@ class SkipTest(Exception): pass +try: + import pytest +except ImportError: + pytest = None + PY3 = sys.version_info[0] == 3 PY2 = sys.version_info[0] == 2 - +PYTEST4 = pytest and pytest.__version__ >= '4.0.0' if PY3: # Python 3 doesn't have an InstanceType, so just use a dummy type. @@ -352,6 +357,120 @@ def __init__(self, input, doc_func=None, skip_on_empty=False): def __call__(self, test_func): self.assert_not_in_testcase_subclass() + input = self.get_input() + wrapper = self._wrap_test_func(test_func, input) + wrapper.parameterized_input = input + wrapper.parameterized_func = test_func + test_func.__name__ = "_parameterized_original_%s" %(test_func.__name__, ) + + return wrapper + + def _wrap_test_func(self, test_func, input): + """ Wraps a test function so that it will appropriately handle + parameterization. + + In the general case, the wrapper will enumerate the input, yielding + test cases. + + In the case of pytest4, the wrapper will use + ``@pytest.mark.parametrize`` to parameterize the test function. """ + + if not input: + if not self.skip_on_empty: + raise ValueError( + "Parameters iterable is empty (hint: use " + "`parameterized([], skip_on_empty=True)` to skip " + "this test when the input is empty)" + ) + return wraps(test_func)(skip_on_empty_helper) + + if PYTEST4: + # pytest >= 4 compatibility is... a bit of work. Basically, the + # only way (I can find) of implementing parameterized testing with + # pytest >= 4 is through the ``@pytest.mark.parameterized`` + # decorator. This decorator has some strange requirements around + # the name and number of arguments to the test function, so this + # wrapper works around that by: + # 1. Introspecting the original test function to determine the + # names and default values of all arguments. + # 2. Creating a new function with the same arguments, but none + # of them are optional:: + # + # def foo(a, b=42): ... + # + # Becomes: + # + # def parameterized_pytest_wrapper_foo(a, b): ... + # + # 3. Merging the ``@parameterized`` parameters with the argument + # default values. + # 4. Generating a list of ``pytest.param(...)`` values, and passing + # that into ``@pytest.mark.parameterized``. + # Some work also needs to be done to support the documented usage + # of ``mock.patch``, which also adds complexity. + Undefined = object() + test_func_wrapped = test_func + test_func_real, mock_patchings = unwrap_mock_patch_func(test_func_wrapped) + func_argspec = getargspec(test_func_real) + + func_args = func_argspec.args + if mock_patchings: + func_args = func_args[:-len(mock_patchings)] + + func_args_no_self = func_args + if func_args_no_self[:1] == ["self"]: + func_args_no_self = func_args_no_self[1:] + + args_with_default = dict( + (arg, Undefined) + for arg in func_args_no_self + ) + for (arg, default) in zip(reversed(func_args_no_self), reversed(func_argspec.defaults or [])): + args_with_default[arg] = default + + pytest_params = [] + for i in input: + p = dict(args_with_default) + for (arg, val) in zip(func_args_no_self, i.args): + p[arg] = val + p.update(i.kwargs) + + # Sanity check: all arguments should now be defined + if any(v is Undefined for v in p.values()): + raise ValueError( + "When parameterizing function %r: no value for " + "arguments: %s with parameters %r " + "(see: 'no value for arguments' in " + "https://github.com/wolever/parameterized#faq)" %( + test_func, + ", ".join( + repr(arg) + for (arg, val) in p.items() + if val is Undefined + ), + i, + ) + ) + + pytest_params.append(pytest.param(*[ + p.get(arg) for arg in func_args_no_self + ])) + + namespace = { + "__parameterized_original_test_func": test_func_wrapped, + } + wrapper_name = "parameterized_pytest_wrapper_%s" %(test_func.__name__, ) + exec( + "def %s(%s, *__args): return __parameterized_original_test_func(%s, *__args)" %( + wrapper_name, + ",".join(func_args), + ",".join(func_args), + ), + namespace, + namespace, + ) + return pytest.mark.parametrize(",".join(func_args_no_self), pytest_params)(namespace[wrapper_name]) + @wraps(test_func) def wrapper(test_self=None): test_cls = test_self and type(test_self) @@ -366,7 +485,7 @@ def wrapper(test_self=None): ) %(test_self, )) original_doc = wrapper.__doc__ - for num, args in enumerate(wrapper.parameterized_input): + for num, args in enumerate(input): p = param.from_decorator(args) unbound_func, nose_tuple = self.param_as_nose_tuple(test_self, test_func, num, p) try: @@ -383,21 +502,6 @@ def wrapper(test_self=None): if test_self is not None: delattr(test_cls, test_func.__name__) wrapper.__doc__ = original_doc - - input = self.get_input() - if not input: - if not self.skip_on_empty: - raise ValueError( - "Parameters iterable is empty (hint: use " - "`parameterized([], skip_on_empty=True)` to skip " - "this test when the input is empty)" - ) - wrapper = wraps(test_func)(skip_on_empty_helper) - - wrapper.parameterized_input = input - wrapper.parameterized_func = test_func - test_func.__name__ = "_parameterized_original_%s" %(test_func.__name__, ) - return wrapper def param_as_nose_tuple(self, test_self, func, num, p): @@ -618,6 +722,11 @@ def decorator(base_class): return decorator +def unwrap_mock_patch_func(f): + if not hasattr(f, "patchings"): + return (f, []) + real_func, patchings = unwrap_mock_patch_func(f.__wrapped__) + return (real_func, patchings + f.patchings) def get_class_name_suffix(params_dict): if "name" in params_dict: diff --git a/parameterized/test.py b/parameterized/test.py index f98d865..894b78b 100644 --- a/parameterized/test.py +++ b/parameterized/test.py @@ -6,8 +6,9 @@ from nose.tools import assert_equal, assert_raises from .parameterized import ( - PY3, PY2, parameterized, param, parameterized_argument_value_pairs, - short_repr, detect_runner, parameterized_class, SkipTest, + PY3, PY2, PYTEST4, parameterized, param, + parameterized_argument_value_pairs, short_repr, detect_runner, + parameterized_class, SkipTest, ) def assert_contains(haystack, needle): @@ -40,6 +41,7 @@ def expect(skip, tests=None): test_params = [ (42, ), + (42, "bar_val"), "foo0", param("foo1"), param("foo2", bar=42), @@ -50,6 +52,7 @@ def expect(skip, tests=None): "test_naked_function('foo1', bar=None)", "test_naked_function('foo2', bar=42)", "test_naked_function(42, bar=None)", + "test_naked_function(42, bar='bar_val')", ]) @parameterized(test_params) @@ -63,6 +66,7 @@ class TestParameterized(object): "test_instance_method('foo1', bar=None)", "test_instance_method('foo2', bar=42)", "test_instance_method(42, bar=None)", + "test_instance_method(42, bar='bar_val')", ]) @parameterized(test_params) @@ -95,10 +99,16 @@ def test_setup(self, count, *a): missing_tests.remove("test_setup(%s)" %(self.actual_order, )) -def custom_naming_func(custom_tag): +def custom_naming_func(custom_tag, kw_name): def custom_naming_func(testcase_func, param_num, param): - return testcase_func.__name__ + ('_%s_name_' % custom_tag) + str(param.args[0]) - + return ( + testcase_func.__name__ + + '_%s_name_' %(custom_tag, ) + + str(param.args[0]) + + # This ... is a bit messy, to properly handle the values in + # `test_params`, but ... it should work. + '_%s' %(param.args[1] if len(param.args) > 1 else param.kwargs.get(kw_name), ) + ) return custom_naming_func @@ -137,19 +147,20 @@ def test_multiple_function_patch_decorator(self, foo, bar, mock_umask, mock_fdopen._mock_name, mock_getpid._mock_name)) -@mock.patch("os.getpid") -class TestParameterizedExpandWithNoExpand(object): - expect("generator", [ - "test_patch_class_no_expand(42, 51, 'umask', 'getpid')", - ]) +if not PYTEST4: + @mock.patch("os.getpid") + class TestParameterizedExpandWithNoExpand(object): + expect("generator", [ + "test_patch_class_no_expand(42, 51, 'umask', 'getpid')", + ]) - @parameterized([(42, 51)]) - @mock.patch("os.umask") - def test_patch_class_no_expand(self, foo, bar, mock_umask, mock_getpid): - missing_tests.remove("test_patch_class_no_expand" - "(%r, %r, %r, %r)" % - (foo, bar, mock_umask._mock_name, - mock_getpid._mock_name)) + @parameterized([(42, 51)]) + @mock.patch("os.umask") + def test_patch_class_no_expand(self, foo, bar, mock_umask, mock_getpid): + missing_tests.remove("test_patch_class_no_expand" + "(%r, %r, %r, %r)" % + (foo, bar, mock_umask._mock_name, + mock_getpid._mock_name)) class TestParameterizedExpandWithNoMockPatchForClass(TestCase): @@ -214,6 +225,7 @@ class TestParamerizedOnTestCase(TestCase): "test_on_TestCase('foo1', bar=None)", "test_on_TestCase('foo2', bar=42)", "test_on_TestCase(42, bar=None)", + "test_on_TestCase(42, bar='bar_val')", ]) @parameterized.expand(test_params) @@ -221,20 +233,21 @@ def test_on_TestCase(self, foo, bar=None): missing_tests.remove("test_on_TestCase(%r, bar=%r)" %(foo, bar)) expect([ - "test_on_TestCase2_custom_name_42(42, bar=None)", - "test_on_TestCase2_custom_name_foo0('foo0', bar=None)", - "test_on_TestCase2_custom_name_foo1('foo1', bar=None)", - "test_on_TestCase2_custom_name_foo2('foo2', bar=42)", + "test_on_TestCase2_custom_name_42_None(42, bar=None)", + "test_on_TestCase2_custom_name_42_bar_val(42, bar='bar_val')", + "test_on_TestCase2_custom_name_foo0_None('foo0', bar=None)", + "test_on_TestCase2_custom_name_foo1_None('foo1', bar=None)", + "test_on_TestCase2_custom_name_foo2_42('foo2', bar=42)", ]) @parameterized.expand(test_params, - name_func=custom_naming_func("custom")) + name_func=custom_naming_func("custom", "bar")) def test_on_TestCase2(self, foo, bar=None): stack = inspect.stack() frame = stack[1] frame_locals = frame[0].f_locals nose_test_method_name = frame_locals['a'][0]._testMethodName - expected_name = "test_on_TestCase2_custom_name_" + str(foo) + expected_name = "test_on_TestCase2_custom_name_" + str(foo) + "_" + str(bar) assert_equal(nose_test_method_name, expected_name, "Test Method name '%s' did not get customized to expected: '%s'" % (nose_test_method_name, expected_name)) @@ -373,6 +386,8 @@ def tearDownModule(): def test_old_style_classes(): if PY3: raise SkipTest("Py3 doesn't have old-style classes") + if PYTEST4: + raise SkipTest("We're not going to worry about old style classes with pytest 4") class OldStyleClass: @parameterized(["foo"]) def parameterized_method(self, param): @@ -552,3 +567,16 @@ class TestUnicodeDocstring(object): def test_with_docstring(self, param): """ Это док-стринг, содержащий не-ascii символы """ pass + +if PYTEST4: + def test_missing_argument_error(): + try: + @parameterized([ + (1, ), + ]) + def foo(a, b): + pass + except ValueError as e: + assert_contains(repr(e), "no value for arguments: 'b'") + else: + raise AssertionError("Expected exception not raised") diff --git a/rebuild-circleci-yaml b/rebuild-circleci-yaml index 5f5224b..bda981e 100755 --- a/rebuild-circleci-yaml +++ b/rebuild-circleci-yaml @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """ Rebuilds the .circleci/config.yml file based on the output of ``tox --listenvs`` @@ -56,12 +56,12 @@ py_version_map = { "py36": "3.6", "py37": "3.7", "py38": "3.8", - "py39": "3.9-rc", + "py39": "3.9", "pypy": "pypy", } def main(): - env_list_str = sp.check_output("tox --listenvs", shell=True).splitlines() + env_list_str = sp.check_output("tox --listenvs", shell=True).decode("utf-8").splitlines() env_list = [] for env in env_list_str: if not env.strip(): diff --git a/tox.ini b/tox.ini index 67a3ccf..37fbca0 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist=py{27,35,36,py}-{nose,nose2,pytest2,pytest3,unit,unit2},py{37,38,39}-{nose,nose2,pytest3,unit,unit2} +envlist=py{27,35,36,py}-{nose,nose2,pytest2,pytest3,pytest4,unit,unit2},py{37,38,39}-{nose,nose2,pytest3,pytest4,unit,unit2} [testenv] deps= nose @@ -7,13 +7,13 @@ deps= nose2: nose2 pytest2: pytest>=2,<3 pytest3: pytest>=3,<4 - #pytest4: pytest>=4,<5 + pytest4: pytest>=4,<5 unit2: unittest2 commands= nose: nosetests nose2: nose2 pytest2: py.test parameterized/test.py pytest3: py.test parameterized/test.py - #pytest4: py.test parameterized/test.py + pytest4: py.test parameterized/test.py unit: python -m unittest parameterized.test unit2: unit2 parameterized.test