Merge pull request #8276 from pfmoore/test_marker

Add a mark for tests that fail on the new resolver

setup.cfg

@@ -39,7 +39,7 @@ ignore_errors = True
 [tool:pytest]
 addopts = --ignore src/pip/_vendor --ignore tests/tests_cache -r aR
 markers =
-    network: tests that needs network
+    network: tests that need network
     incompatible_with_test_venv
     incompatible_with_venv
     no_auto_tempdir_manager
@@ -50,6 +50,7 @@ markers =
     mercurial: VCS: Mercurial
     git: VCS: git
     yaml: yaml based tests
+    fails_on_new_resolver: Does not yet work on the new resolver
 
 [bdist_wheel]
 universal = 1

tests/conftest.py

@@ -41,6 +41,12 @@ def pytest_addoption(parser):
         default=False,
         help="use new resolver in tests",
     )
+    parser.addoption(
+        "--new-resolver-runtests",
+        action="store_true",
+        default=False,
+        help="run the skipped tests for the new resolver",
+    )
     parser.addoption(
         "--use-venv",
         action="store_true",
@@ -59,6 +65,12 @@ def pytest_collection_modifyitems(config, items):
                 "CI" in os.environ):
             item.add_marker(pytest.mark.flaky(reruns=3))
 
+        if (item.get_closest_marker('fails_on_new_resolver') and
+                config.getoption("--new-resolver") and
+                not config.getoption("--new-resolver-runtests")):
+            item.add_marker(pytest.mark.skip(
+                'This test does not work with the new resolver'))
+
         if six.PY3:
             if (item.get_closest_marker('incompatible_with_test_venv') and
                     config.getoption("--use-venv")):

tests/functional/test_cache.py

@@ -170,7 +170,7 @@ def test_cache_list_name_and_version_match(script):
     assert not list_matches_wheel('zzz-7.8.9', result)
 
 
-@pytest.mark.usefixture("populate_wheel_cache")
+@pytest.mark.usefixtures("populate_wheel_cache")
 def test_cache_remove_no_arguments(script):
     """Running `pip cache remove` with no arguments should cause an error."""
     script.pip('cache', 'remove', expect_error=True)

tests/functional/test_download.py

@@ -669,6 +669,7 @@ def test_download_exit_status_code_when_blank_requirements_file(script):
     script.pip('download', '-r', 'blank.txt')
 
 
+@pytest.mark.fails_on_new_resolver
 def test_download_prefer_binary_when_tarball_higher_than_wheel(script, data):
     fake_wheel(data, 'source-0.8-py2.py3-none-any.whl')
     result = script.pip(

tests/functional/test_install.py

@@ -148,6 +148,7 @@ def test_pep518_with_user_pip(script, pip_src, data, common_wheels):
     )
 
 
+@pytest.mark.fails_on_new_resolver
 def test_pep518_with_extra_and_markers(script, data, common_wheels):
     script.pip(
         'wheel', '--no-index',
@@ -532,6 +533,7 @@ def assert_re_match(pattern, text):
 
 
 @pytest.mark.network
+@pytest.mark.fails_on_new_resolver
 def test_hashed_install_failure_later_flag(script, tmpdir):
     with requirements_file(
         "blessings==1.0\n"
@@ -937,6 +939,7 @@ def test_install_nonlocal_compatible_wheel(script, data):
     assert result.returncode == ERROR
 
 
+@pytest.mark.fails_on_new_resolver
 def test_install_nonlocal_compatible_wheel_path(script, data):
     target_dir = script.scratch_path / 'target'
 
@@ -1491,6 +1494,7 @@ def test_double_install(script):
     assert msg not in result.stderr
 
 
+@pytest.mark.fails_on_new_resolver
 def test_double_install_fail(script):
     """
     Test double install failing with two different version requirements
@@ -1746,6 +1750,7 @@ def test_user_config_accepted(script):
     ]
 )
 @pytest.mark.parametrize("use_module", [True, False])
+@pytest.mark.fails_on_new_resolver
 def test_install_pip_does_not_modify_pip_when_satisfied(
         script, install_args, expected_message, use_module):
     """
@@ -1757,6 +1762,7 @@ def test_install_pip_does_not_modify_pip_when_satisfied(
     assert expected_message in result.stdout, str(result)
 
 
+@pytest.mark.fails_on_new_resolver
 def test_ignore_yanked_file(script, data):
     """
     Test ignore a "yanked" file.
@@ -1794,6 +1800,7 @@ def test_valid_index_url_argument(script, shared_data):
     assert 'Successfully installed Dinner' in result.stdout, str(result)
 
 
+@pytest.mark.fails_on_new_resolver
 def test_install_yanked_file_and_print_warning(script, data):
     """
     Test install a "yanked" file and print a warning.
@@ -1873,6 +1880,7 @@ def test_install_skip_work_dir_pkg(script, data):
     assert 'Successfully installed simple' in result.stdout
 
 
+@pytest.mark.fails_on_new_resolver
 def test_install_include_work_dir_pkg(script, data):
     """
     Test that install of a package in working directory

tests/functional/test_install_config.py

@@ -7,6 +7,7 @@
 from tests.lib.server import file_response, package_page
 
 
+@pytest.mark.fails_on_new_resolver
 def test_options_from_env_vars(script):
     """
     Test if ConfigOptionParser reads env vars (e.g. not using PyPI here)
@@ -43,6 +44,7 @@ def test_command_line_options_override_env_vars(script, virtualenv):
 
 
 @pytest.mark.network
+@pytest.mark.fails_on_new_resolver
 def test_env_vars_override_config_file(script, virtualenv):
     """
     Test that environmental variables override settings in config files.
@@ -174,6 +176,7 @@ def test_config_file_override_stack(
     assert requests[3]["PATH_INFO"] == "/files/INITools-0.2.tar.gz"
 
 
+@pytest.mark.fails_on_new_resolver
 def test_options_from_venv_config(script, virtualenv):
     """
     Test if ConfigOptionParser reads a virtualenv-local config file

tests/functional/test_install_direct_url.py

@@ -1,5 +1,7 @@
 import re
 
+import pytest
+
 from pip._internal.models.direct_url import DIRECT_URL_METADATA_NAME, DirectUrl
 from tests.lib import _create_test_package, path_to_url
 
@@ -30,6 +32,7 @@ def test_install_vcs_editable_no_direct_url(script, with_wheel):
     assert not _get_created_direct_url(result, "testpkg")
 
 
+@pytest.mark.fails_on_new_resolver
 def test_install_vcs_non_editable_direct_url(script, with_wheel):
     pkg_path = _create_test_package(script, name="testpkg")
     url = path_to_url(pkg_path)

tests/functional/test_install_extras.py

@@ -136,6 +136,7 @@ def test_install_special_extra(script):
         pytest.param('[extra2]', '1.0', marks=pytest.mark.xfail),
         pytest.param('[extra1,extra2]', '1.0', marks=pytest.mark.xfail),
     ])
+@pytest.mark.fails_on_new_resolver
 def test_install_extra_merging(script, data, extra_to_install, simple_version):
     # Check that extra specifications in the extras section are honoured.
     pkga_path = script.scratch_path / 'pkga'

tests/functional/test_install_reqs.py

@@ -189,6 +189,7 @@ def test_respect_order_in_requirements_file(script, data):
     )
 
 
+@pytest.mark.fails_on_new_resolver
 def test_install_local_editable_with_extras(script, data):
     to_install = data.packages.joinpath("LocalExtras")
     res = script.pip_install_local(
@@ -336,6 +337,7 @@ def test_constraints_local_install_causes_error(script, data):
     assert 'Could not satisfy constraints for' in result.stderr
 
 
+@pytest.mark.fails_on_new_resolver
 def test_constraints_constrain_to_local_editable(script, data):
     to_install = data.src.joinpath("singlemodule")
     script.scratch_path.joinpath("constraints.txt").write_text(
@@ -347,6 +349,7 @@ def test_constraints_constrain_to_local_editable(script, data):
     assert 'Running setup.py develop for singlemodule' in result.stdout
 
 
+@pytest.mark.fails_on_new_resolver
 def test_constraints_constrain_to_local(script, data):
     to_install = data.src.joinpath("singlemodule")
     script.scratch_path.joinpath("constraints.txt").write_text(
@@ -358,6 +361,7 @@ def test_constraints_constrain_to_local(script, data):
     assert 'Running setup.py install for singlemodule' in result.stdout
 
 
+@pytest.mark.fails_on_new_resolver
 def test_constrained_to_url_install_same_url(script, data):
     to_install = data.src.joinpath("singlemodule")
     constraints = path_to_url(to_install) + "#egg=singlemodule"
@@ -403,6 +407,7 @@ def test_double_install_spurious_hash_mismatch(
         assert 'Successfully installed simple-1.0' in str(result)
 
 
+@pytest.mark.fails_on_new_resolver
 def test_install_with_extras_from_constraints(script, data):
     to_install = data.packages.joinpath("LocalExtras")
     script.scratch_path.joinpath("constraints.txt").write_text(
@@ -413,6 +418,7 @@ def test_install_with_extras_from_constraints(script, data):
     assert script.site_packages / 'simple' in result.files_created
 
 
+@pytest.mark.fails_on_new_resolver
 def test_install_with_extras_from_install(script, data):
     to_install = data.packages.joinpath("LocalExtras")
     script.scratch_path.joinpath("constraints.txt").write_text(
@@ -423,6 +429,7 @@ def test_install_with_extras_from_install(script, data):
     assert script.site_packages / 'singlemodule.py' in result.files_created
 
 
+@pytest.mark.fails_on_new_resolver
 def test_install_with_extras_joined(script, data):
     to_install = data.packages.joinpath("LocalExtras")
     script.scratch_path.joinpath("constraints.txt").write_text(
@@ -435,6 +442,7 @@ def test_install_with_extras_joined(script, data):
     assert script.site_packages / 'singlemodule.py' in result.files_created
 
 
+@pytest.mark.fails_on_new_resolver
 def test_install_with_extras_editable_joined(script, data):
     to_install = data.packages.joinpath("LocalExtras")
     script.scratch_path.joinpath("constraints.txt").write_text(
@@ -465,6 +473,7 @@ def test_install_distribution_duplicate_extras(script, data):
         assert expected in result.stderr
 
 
+@pytest.mark.fails_on_new_resolver
 def test_install_distribution_union_with_constraints(script, data):
     to_install = data.packages.joinpath("LocalExtras")
     script.scratch_path.joinpath("constraints.txt").write_text(
@@ -475,6 +484,7 @@ def test_install_distribution_union_with_constraints(script, data):
     assert script.site_packages / 'singlemodule.py' in result.files_created
 
 
+@pytest.mark.fails_on_new_resolver
 def test_install_distribution_union_with_versions(script, data):
     to_install_001 = data.packages.joinpath("LocalExtras")
     to_install_002 = data.packages.joinpath("LocalExtras-0.0.2")
@@ -497,6 +507,7 @@ def test_install_distribution_union_conflicting_extras(script, data):
     assert "Conflict" in result.stderr
 
 
+@pytest.mark.fails_on_new_resolver
 def test_install_unsupported_wheel_link_with_marker(script):
     script.scratch_path.joinpath("with-marker.txt").write_text(
         textwrap.dedent("""\
@@ -515,6 +526,7 @@ def test_install_unsupported_wheel_link_with_marker(script):
     assert len(result.files_created) == 0
 
 
+@pytest.mark.fails_on_new_resolver
 def test_install_unsupported_wheel_file(script, data):
     # Trying to install a local wheel with an incompatible version/type
     # should fail.

tests/functional/test_install_upgrade.py

@@ -36,6 +36,7 @@ def test_invalid_upgrade_strategy_causes_error(script):
     assert "invalid choice" in result.stderr
 
 
+@pytest.mark.fails_on_new_resolver
 def test_only_if_needed_does_not_upgrade_deps_when_satisfied(script):
     """
     It doesn't upgrade a dependency if it already satisfies the requirements.
@@ -181,6 +182,7 @@ def test_upgrade_if_requested(script):
     )
 
 
+@pytest.mark.fails_on_new_resolver
 def test_upgrade_with_newest_already_installed(script, data):
     """
     If the newest version of a package is already installed, the package should
@@ -249,6 +251,7 @@ def test_uninstall_before_upgrade_from_url(script):
 
 
 @pytest.mark.network
+@pytest.mark.fails_on_new_resolver
 def test_upgrade_to_same_version_from_url(script):
     """
     When installing from a URL the same version that is already installed, no

tests/functional/test_install_user.py

@@ -126,6 +126,7 @@ def test_install_user_conflict_in_usersite(self, script):
 
     @pytest.mark.network
     @pytest.mark.incompatible_with_test_venv
+    @pytest.mark.fails_on_new_resolver
     def test_install_user_conflict_in_globalsite(self, virtualenv, script):
         """
         Test user install with conflict in global site ignores site and
@@ -158,6 +159,7 @@ def test_install_user_conflict_in_globalsite(self, virtualenv, script):
 
     @pytest.mark.network
     @pytest.mark.incompatible_with_test_venv
+    @pytest.mark.fails_on_new_resolver
     def test_upgrade_user_conflict_in_globalsite(self, virtualenv, script):
         """
         Test user install/upgrade with conflict in global site ignores site and
@@ -189,6 +191,7 @@ def test_upgrade_user_conflict_in_globalsite(self, virtualenv, script):
 
     @pytest.mark.network
     @pytest.mark.incompatible_with_test_venv
+    @pytest.mark.fails_on_new_resolver
     def test_install_user_conflict_in_globalsite_and_usersite(
             self, virtualenv, script):
         """

tests/functional/test_install_vcs_git.py

@@ -495,6 +495,7 @@ def test_install_git_branch_not_cached(script, with_wheel):
     ), result.stdout
 
 
+@pytest.mark.fails_on_new_resolver
 def test_install_git_sha_cached(script, with_wheel):
     """
     Installing git urls with a sha revision does cause wheel caching.

tests/functional/test_uninstall_user.py

@@ -22,6 +22,7 @@ def test_uninstall_from_usersite(self, script):
         result2 = script.pip('uninstall', '-y', 'INITools')
         assert_all_changes(result1, result2, [script.venv / 'build', 'cache'])
 
+    @pytest.mark.fails_on_new_resolver
     def test_uninstall_from_usersite_with_dist_in_global_site(
             self, virtualenv, script):
         """

tests/functional/test_wheel.py

@@ -62,6 +62,7 @@ def test_pip_wheel_success(script, data):
     assert "Successfully built simple" in result.stdout, result.stdout
 
 
+@pytest.mark.fails_on_new_resolver
 def test_pip_wheel_build_cache(script, data):
     """
     Test 'pip wheel' builds and caches.