ENT-4252: Migrate from nose to pytest

* Card ID: ENT-4252

This commit replaces nosetests with pytest as a test suite.

The DBus tests have been disabled, because they fail when not run under
nosetests, and will have to be fixed before they are enabled again. They
can be selected manually by using the '-m dbus' flag.

.coveragerc

@@ -1,9 +1,4 @@
 [run]
 branch = True
-source = subscription_manager
-         rct
-         rhsm
-         rhsm_debug
-         content_plugins
-[report]
-include = src/*
+source = src/
+command_line = -m pytest test/

.gitignore

@@ -4,6 +4,7 @@
 *.so
 *.o
 .idea*
+.pytest_cache
 *.coverage
 .ropeproject
 *.cov2emacs.log

Jenkinsfile

@@ -21,7 +21,7 @@ pipeline {
         stage('Fedora unit') {
           steps {
             sh readFile(file: 'jenkins/python3-tests.sh')
-            junit('nosetests.xml')
+            junit('coverage.xml')
             // TODO: find the correct adapter or generate coverage tests that can be
             //       parsed by an existing adapter:
             //       https://plugins.jenkins.io/code-coverage-api/

Makefile

@@ -260,7 +260,7 @@ install-files: dbus-install install-conf install-plugins
 
 .PHONY: check
 check:
-	$(PYTHON) setup.py -q nosetests -c playpen/noserc.dev
+	pytest test/
 
 .PHONY: version_check
 version_check:
@@ -274,9 +274,9 @@ coverage:
 ifdef ghprbPullId
 	# Pull the PR id from the Jenkins environment and use it as a seed so that each PR
 	# uses a consistant test ordering.
-	$(PYTHON) ./setup.py -q nosetests --randomly-seed=$(ghprbPullId) -c playpen/noserc.ci
+	$(PYTHON) -m coverage run -m pytest --randomly-seed=$(ghprbPullId) test/
 else
-	$(PYTHON) ./setup.py -q nosetests -c playpen/noserc.ci
+	$(PYTHON) -m coverage run -m pytest test/
 endif
 
 .PHONY: docs

README.md

@@ -241,14 +241,7 @@ See `cockpit/README.md` for more detailed information on cockpit development.
 
 Testing
 -------
-We run tests using nose (see candlepinproject.org for details).  Some tests
-are not run by default. For example, since we are not maintaining the GTK GUI
-for all platforms, they are not run by default. They can be included via
-`-a gui` option for the nose command. It is recommended if you run the GUI
-tests to also use `--with-xvfb` in order to use Xvfb instead of spawning
-GTK windows in your desktop session (ex. `nosetests -a gui --with-xvfb`).
-
-[More details about testing](./TESTING.md)
+We run tests using pytest. [See TESTING.md for more details.](./TESTING.md)
 
 Troubleshooting
 ---------------

TESTING.md

@@ -1,3 +1,75 @@
+# TESTING
+
+- [subscription-manager](#subscription-manager)
+
+## subscription-manager
+
+```bash
+pytest
+# or, for increased verbosity
+pytest -v
+```
+
+Some tests are disabled by default, some can be disabled manually. Defaults are stored in the `pytest.ini` file. To overwrite them, use `-m` argument.
+
+For example, to run just the Zypper tests, execute
+
+```bash
+pytest -m "functional and zypper"
+```
+
+or, to disable DBus tests, run
+
+```bash
+pytest -m "not dbus"
+```
+
+To test specific class or function, use `::` as separator:
+
+```bash
+pytest test/test_i18n.py::TestI18N::test_text_width
+```
+
+To only run tests containing some substring, run
+
+```bash
+pytest -k cache
+# or, to omit the summary, run
+pytest -k cache --no-summary
+```
+
+### Plugins
+
+- To disable pytest-randomly plugin, run
+
+```bash
+pytest -p no:randomly test/
+```
+
+- If you install `pytest-xdist` the tests can be run in parallel. The following runs in 9.67s instead of 22.41s:
+
+```bash
+pytest -n 4 --no-summary -p no:randomly -v test/
+```
+
+After all the tests are run, a warnings summary is displayed with the list of deprecations. It can be disabled with `--disable-warnings`. Whole summary can be disabled with `--no-summary`.
+
+To compute coverage, run
+
+```bash
+coverage run
+# display ASCII report
+coverage report
+# generate interactive HTML report to htmlcov/
+coverage html
+# generate XML report readable by tools like IDEs
+# - PyCharm: Run > Show Coverage Data (Ctrl+Alt+6)
+coverage xml
+```
+
+---
+
+Following text may not be up to date.
 
 # Testing of the game
 

build_ext/setup.py

@@ -15,7 +15,9 @@
 from setuptools import setup, find_packages
 
 test_require = [
-    'nose',
+    'pytest',
+    'pytest-randomly',
+    'pytest-timeout',
     'mock'
 ]
 
@@ -35,5 +37,4 @@
     packages=find_packages(),
     tests_require=test_require,
     install_requires=install_requires,
-    test_suite='nose.collector',
 )

jenkins/python3-tests.sh

@@ -29,13 +29,8 @@ pip install -I -r test-requirements.txt
 python3 setup.py build
 python3 setup.py build_ext --inplace
 
-# not using "setup.py nosetests" yet
-# since they need a running candlepin
-
-# Run just the unit tests, functional needs a running candlepin
-#pushd test/unit
 # make sure we have a dbus session for the dbus tests
-dbus-run-session nosetests --with-xunit --with-cover --cover-package rhsm --cover-package subscription_manager --cover-erase
+dbus-run-session coverage run
 
-coverage3 html
-coverage3 xml
+coverage report
+coverage xml

pytest.ini

@@ -0,0 +1,21 @@
+[pytest]
+addopts =
+	# Non-registered markers raise errors
+	--strict-markers
+	# Start with tests that failed last time, --ff
+	--failed-first
+	# Show extra summary for (f)ailed, (E)rror, (s)kipped and (w)arnings
+	-rfEsw
+	# Mark some tests not to be run
+	-m "not dbus and not functional"
+markers =
+	dbus: subscription-manager tests for DBus.
+	slow: subscription-manager tests that may be slower than the rest.
+	zypper: subscription-manager tests for the Zypper package manager.
+	functional: subscription-manager functional tests.
+timeout = 5
+testpaths =
+	test/
+required_plugins =
+	pytest-randomly
+	pytest-timeout

setup.py

@@ -284,9 +284,9 @@ def find_py(self):
 
 test_require = [
     'mock',
-    'nose',
-    'nose-capturestderr',
-    'nose-randomly',
+    'pytest',
+    'pytest-randomly',
+    'pytest-timeout',
     'coverage',
     'polib',
     'flake8',
@@ -359,5 +359,4 @@ def find_py(self):
     tests_require=test_require,
     ext_modules=[Extension('rhsm._certificate', ['src/certificate.c'],
                            libraries=['ssl', 'crypto'])],
-    test_suite='nose.collector',
 )

test-requirements.txt

@@ -5,14 +5,13 @@
 # immediately available.
 -e ./build_ext
 
-nose
-nose-capturestderr
-nose-randomly
+pytest
+pytest-randomly
+pytest-timeout
 coverage
 polib
 pyinotify
 simplejson
 mock
 Sphinx==1.8.5;python_version<="2.7"
 Sphinx>=1.8.5;python_version>="3.0"
-git+https://github.com/awood/nose-xvfb.git#egg=nose-xvfb

test/__init__.py

@@ -1,5 +1,32 @@
 from __future__ import print_function, division, absolute_import
 
+import os
+import pathlib
+import sys
+from typing import Callable, List
 
 from . import rhsm_display
 rhsm_display.set_display()
+
+# Hijack sys.path, so we don't have to use 'PYTHONPATH=src/' prefix
+rootdir = pathlib.Path(__file__).parent.parent
+sys.path.insert(0, str(rootdir / "src"))
+
+import pytest
+
+subman_marker_dbus = pytest.mark.dbus
+subman_marker_functional = pytest.mark.functional
+subman_marker_zypper = pytest.mark.zypper
+subman_marker_slow = pytest.mark.slow
+# This allows us to set higher timeout limit for tests that are known to be slow
+subman_marker_slow_timeout = pytest.mark.timeout(40)
+
+
+def subman_marker_needs_envvars(*envvars: List[str]) -> Callable:
+    """Skip test if one or more environment variables are missing."""
+    missing_vars: List[str] = [v for v in envvars if v not in os.environ]
+    skip_func: Callable = pytest.mark.skipif(
+        missing_vars,
+        reason=f"Missing environment variables {', '.join(missing_vars)}."
+    )
+    return skip_func

test/rhsm/functional/connection_tests.py

@@ -20,7 +20,7 @@
 import string
 import unittest
 
-from nose.plugins.attrib import attr
+from test import subman_marker_functional
 
 from rhsm.connection import ContentConnection, UEPConnection, Restlib,\
     UnauthorizedException, ForbiddenException, RestlibException
@@ -34,7 +34,7 @@ def random_string(name, target_length=32):
     return name
 
 
-@attr('functional')
+@subman_marker_functional
 class ConnectionTests(unittest.TestCase):
 
     def setUp(self):
@@ -143,7 +143,7 @@ def tearDown(self):
         self.cp.unregisterConsumer(self.consumer_uuid)
 
 
-@attr('functional')
+@subman_marker_functional
 class EntitlementRegenerationTests(unittest.TestCase):
     def setUp(self):
         self.cp = UEPConnection(username="admin", password="admin", insecure=True)
@@ -200,7 +200,7 @@ def tearDown(self):
         self.cp.unregisterConsumer(self.consumer_uuid)
 
 
-@attr('functional')
+@subman_marker_functional
 class BindRequestTests(unittest.TestCase):
     def setUp(self):
         self.cp = UEPConnection(username="admin", password="admin", insecure=True)
@@ -235,7 +235,7 @@ def test_bind_by_pool(self, mock_conn, mock_drift, mock_validate):
                 self.assertEqual(None, kwargs['body'])
 
 
-@attr('functional')
+@subman_marker_functional
 class ContentConnectionTests(unittest.TestCase):
 
     def testInsecure(self):
@@ -290,7 +290,7 @@ def testEnvNoProxyWithAsterisk(self):
         assert 'no_proxy' not in os.environ and 'https_proxy' not in os.environ
 
 
-@attr('functional')
+@subman_marker_functional
 class HypervisorCheckinTests(unittest.TestCase):
 
     def setUp(self):
@@ -307,7 +307,7 @@ def test_hypervisor_checkin_can_pass_empty_map_and_updates_nothing(self):
             self.assertEqual(len(response['created']), 0)
 
 
-@attr('functional')
+@subman_marker_functional
 class RestlibTests(unittest.TestCase):
 
     def setUp(self):
@@ -376,7 +376,7 @@ def _check_for_remote_server_exception(self, expected_error_code,
             self.assertEqual(expected_error_code, ex.code)
 
 
-@attr('functional')
+@subman_marker_functional
 class OwnerInfoTests(unittest.TestCase):
     def setUp(self):
         self.cp = UEPConnection(username="admin", password="admin",

test/rhsm/functional/profile_tests.py

@@ -19,10 +19,10 @@
 from rhsm import ourjson as json
 from mock import Mock
 
-from nose.plugins.attrib import attr
+from test import subman_marker_functional
 
 
-@attr('functional')
+@subman_marker_functional
 class ProfileTests(unittest.TestCase):
 
     def test_get_rpm_profile(self):

test/rhsmlib_test/base.py

@@ -108,7 +108,7 @@ def setUp(self):
         self.addCleanup(self.stop_server)
         sender_patcher = mock.patch("rhsmlib.client_info.DBusSender.get_cmd_line")
         self.mock_sender_get_cmd_line = sender_patcher.start()
-        self.mock_sender_get_cmd_line.return_value = "nose-unit-test"
+        self.mock_sender_get_cmd_line.return_value = "unit-test"
 
     def stop_server(self):
         self.server_thread.stop()