From 5d68747b6372a9a984314f6df859a4482feb459d Mon Sep 17 00:00:00 2001 From: Vipul Cariappa Date: Wed, 22 Jan 2025 13:23:21 +0530 Subject: [PATCH] Debug XFails --- .github/workflows/MacOS.yml | 38 +++-------------------------------- .github/workflows/Ubuntu.yml | 16 ++------------- test/support.py | 9 +++++++++ test/test_aclassloader.py | 4 +++- test/test_advancedcpp.py | 5 ++++- test/test_api.py | 4 +++- test/test_boost.py | 5 ++++- test/test_concurrent.py | 4 +++- test/test_conversions.py | 5 ++++- test/test_cpp11features.py | 4 +++- test/test_crossinheritance.py | 4 +++- test/test_datatypes.py | 4 +++- test/test_doc_features.py | 5 ++++- test/test_eigen.py | 5 ++++- test/test_fragile.py | 5 ++++- test/test_leakcheck.py | 5 ++++- test/test_lowlevel.py | 5 ++++- test/test_numba.py | 5 ++++- test/test_operators.py | 5 ++++- test/test_overloads.py | 5 ++++- test/test_pythonify.py | 5 ++++- test/test_pythonization.py | 4 +++- test/test_regression.py | 3 ++- test/test_stltypes.py | 5 ++++- test/test_streams.py | 5 ++++- test/test_templates.py | 5 ++++- 26 files changed, 97 insertions(+), 72 deletions(-) diff --git a/.github/workflows/MacOS.yml b/.github/workflows/MacOS.yml index 3f902ef3..a1fd2995 100644 --- a/.github/workflows/MacOS.yml +++ b/.github/workflows/MacOS.yml @@ -559,7 +559,7 @@ jobs: run: | # Run the tests source ${{ github.workspace }}/CppInterOp/.venv/bin/activate - cd ${{ github.workspace }}/test/ + cd ${{ github.workspace }}/test echo ::group::Prepare For Testing make all python -m pip install --upgrade pip @@ -567,44 +567,12 @@ jobs: python -m pip install pytest-xdist python -m pip install numba echo ::endgroup:: - echo ::group::Run complete test suite - set -o pipefail - python -m pytest -sv -ra | tee complete_testrun.log 2>&1 - set +o pipefail echo ::group::Crashing Test Logs - # See if we don't have a crash that went away - # Comment out all xfails but the ones that have a run=False condition. - find . -name "*.py" -exec sed -i '/run=False/!s/^ *@mark.xfail\(.*\)/#&/' {} \; - python -m pytest -n 1 -m "xfail" --runxfail -sv -ra --max-worker-restart 512 | tee test_crashed.log 2>&1 || true - git checkout . + python -m pytest -n 1 -v -ra --max-worker-restart 512 | tee test_crashed.log 2>&1 || true echo ::endgroup:: - echo ::group::XFAIL Test Logs - # Rewrite all xfails that have a run clause to skipif. This way we will - # avoid conditionally crashing xfails - find . -name "*.py" -exec gsed -i -E 's/(^ *)@mark.xfail\(run=(.*)/\1@mark.skipif(condition=not \2/g' {} \; - # See if we don't have an xfail that went away - python -m pytest --runxfail -sv -ra | tee test_xfailed.log 2>&1 || true - git checkout . - echo ::endgroup:: - echo ::group::Passing Test Logs - - # Run the rest of the non-crashing tests. - declare -i RETCODE=0 - - set -o pipefail - export RETCODE=+$? - echo ::endgroup:: - - RETCODE=+$? - echo "Complete Test Suite Summary: \n" - tail -n1 complete_testrun.log - echo "Crashing Summary: \n" + echo "Crashing Summary:" tail -n1 test_crashed.log - echo "XFAIL Summary:" - tail -n1 test_xfailed.log - echo "Return Code: ${RETCODE}" - exit $RETCODE - name: Show debug info if: ${{ failure() }} diff --git a/.github/workflows/Ubuntu.yml b/.github/workflows/Ubuntu.yml index b0abc601..25eae62b 100644 --- a/.github/workflows/Ubuntu.yml +++ b/.github/workflows/Ubuntu.yml @@ -670,10 +670,6 @@ jobs: python -m uv pip install numba python -m uv pip install psutil echo ::endgroup:: - echo ::group::Run complete test suite - set -o pipefail - python -m pytest -sv -ra | tee complete_testrun.log 2>&1 - set +o pipefail echo ::group::Crashing Test Logs # See if we don't have a crash that went away # Comment out all xfails but the ones that have a run=False condition. @@ -705,20 +701,12 @@ jobs: SUPPRESSION_FILE="../etc/clang${CLANG_VERSION}-valgrind.supp" fi - valgrind --show-error-list=yes --error-exitcode=1 --track-origins=yes --gen-suppressions=all --suppressions="${SUPPRESSION_FILE}" --suppressions=../etc/valgrind-cppyy-cling.supp python -m pytest -m "not xfail" -sv -ra --ignore=test_leakcheck.py + valgrind --show-error-list=yes --error-exitcode=1 --track-origins=yes --gen-suppressions=all --suppressions="${SUPPRESSION_FILE}" --suppressions=../etc/valgrind-cppyy-cling.supp python -m pytest -m "not xfail" -sv -ra export RETCODE=+$? echo ::endgroup:: - RETCODE=+$? - - echo "Complete Test Suite Summary: \n" - tail -n1 complete_testrun.log - echo "Crashing Summary: \n" + echo "Crashing Summary:" tail -n1 test_crashed.log - echo "XFAIL Summary:" - tail -n1 test_xfailed.log - echo "Return Code: ${RETCODE}" - exit $RETCODE - name: Show debug info if: ${{ failure() }} diff --git a/test/support.py b/test/support.py index b69ca309..e7d0e8e2 100644 --- a/test/support.py +++ b/test/support.py @@ -74,3 +74,12 @@ def setup_make(targetname): true #endif\n""") == 1) IS_CLING = not IS_CLANG_REPL + +from pytest import mark + +proxy = mark.xfail +def monkey_patch(*args, **kwargs): + if "run" in kwargs: + del kwargs["run"] + + return proxy(*args, **kwargs) diff --git a/test/test_aclassloader.py b/test/test_aclassloader.py index 16c21176..c08d4368 100644 --- a/test/test_aclassloader.py +++ b/test/test_aclassloader.py @@ -1,6 +1,8 @@ import py, os, sys from pytest import raises, mark -from .support import setup_make +from .support import setup_make, monkey_patch + +mark.xfail = monkey_patch currpath = py.path.local(__file__).dirpath() test_dct = str(currpath.join("example01Dict")) diff --git a/test/test_advancedcpp.py b/test/test_advancedcpp.py index 623452da..6db90f64 100644 --- a/test/test_advancedcpp.py +++ b/test/test_advancedcpp.py @@ -1,6 +1,9 @@ import py, os, sys from pytest import raises, skip, mark -from .support import setup_make, pylong, IS_WINDOWS, IS_MAC, IS_LINUX, ispypy, IS_CLANG_REPL, IS_MAC_ARM, IS_MAC_X86 +from .support import setup_make, pylong, IS_WINDOWS, IS_MAC, IS_LINUX, ispypy, IS_CLANG_REPL, IS_MAC_ARM, IS_MAC_X86, monkey_patch + +mark.xfail = monkey_patch + currpath = py.path.local(__file__).dirpath() test_dct = str(currpath.join("advancedcppDict")) diff --git a/test/test_api.py b/test/test_api.py index 24509945..ab7da4f0 100644 --- a/test/test_api.py +++ b/test/test_api.py @@ -1,6 +1,8 @@ import py, os, sys from pytest import raises, skip, mark -from .support import ispypy, IS_MAC, IS_LINUX_ARM +from .support import ispypy, IS_MAC, IS_LINUX_ARM, monkey_patch + +mark.xfail = monkey_patch class TestAPI: diff --git a/test/test_boost.py b/test/test_boost.py index a3eb38f1..7ba3f197 100644 --- a/test/test_boost.py +++ b/test/test_boost.py @@ -1,6 +1,9 @@ import py, os, sys from pytest import mark, raises, skip -from .support import setup_make, IS_CLANG_REPL, IS_MAC_X86, IS_MAC_ARM +from .support import setup_make, IS_CLANG_REPL, IS_MAC_X86, IS_MAC_ARM, monkey_patch + +mark.xfail = monkey_patch + noboost = False if not (os.path.exists(os.path.join(os.path.sep, 'usr', 'include', 'boost')) or \ diff --git a/test/test_concurrent.py b/test/test_concurrent.py index 5d74d0cd..31f87504 100644 --- a/test/test_concurrent.py +++ b/test/test_concurrent.py @@ -1,6 +1,8 @@ import py, os, sys from pytest import raises, skip, mark -from .support import IS_MAC_ARM, IS_MAC_X86, IS_LINUX_ARM +from .support import IS_MAC_ARM, IS_MAC_X86, IS_LINUX_ARM, monkey_patch + +mark.xfail = monkey_patch class TestCONCURRENT: diff --git a/test/test_conversions.py b/test/test_conversions.py index c9783df2..20a2bccc 100644 --- a/test/test_conversions.py +++ b/test/test_conversions.py @@ -1,6 +1,9 @@ import py, os, sys from pytest import raises, mark -from .support import setup_make, IS_LINUX, IS_CLANG_REPL, IS_CLING, IS_MAC +from .support import setup_make, IS_LINUX, IS_CLANG_REPL, IS_CLING, IS_MAC, monkey_patch + +mark.xfail = monkey_patch + currpath = py.path.local(__file__).dirpath() test_dct = str(currpath.join("conversionsDict")) diff --git a/test/test_cpp11features.py b/test/test_cpp11features.py index f19299e4..11649919 100644 --- a/test/test_cpp11features.py +++ b/test/test_cpp11features.py @@ -1,6 +1,8 @@ import py, os, sys from pytest import raises, mark -from .support import setup_make, ispypy, IS_CLANG_REPL, IS_LINUX_ARM, IS_LINUX, IS_MAC, IS_CLING +from .support import setup_make, ispypy, IS_CLANG_REPL, IS_LINUX_ARM, IS_LINUX, IS_MAC, IS_CLING, monkey_patch + +mark.xfail = monkey_patch currpath = py.path.local(__file__).dirpath() diff --git a/test/test_crossinheritance.py b/test/test_crossinheritance.py index 9bab6994..2b5d9437 100644 --- a/test/test_crossinheritance.py +++ b/test/test_crossinheritance.py @@ -1,6 +1,8 @@ import py, os, sys from pytest import raises, skip, mark -from .support import setup_make, pylong, IS_MAC_ARM, IS_MAC, IS_CLANG_REPL, IS_CLANG_DEBUG, IS_LINUX_ARM, IS_LINUX +from .support import setup_make, pylong, IS_MAC_ARM, IS_MAC, IS_CLANG_REPL, IS_CLANG_DEBUG, IS_LINUX_ARM, IS_LINUX, monkey_patch + +mark.xfail = monkey_patch currpath = py.path.local(__file__).dirpath() diff --git a/test/test_datatypes.py b/test/test_datatypes.py index 3315d9a8..9d1de9b0 100644 --- a/test/test_datatypes.py +++ b/test/test_datatypes.py @@ -1,6 +1,8 @@ import py, os, sys from pytest import raises, skip, mark -from .support import setup_make, pylong, pyunicode, IS_CLANG_REPL, IS_CLING, IS_MAC_X86, IS_MAC_ARM, IS_MAC, IS_LINUX +from .support import setup_make, pylong, pyunicode, IS_CLANG_REPL, IS_CLING, IS_MAC_X86, IS_MAC_ARM, IS_MAC, IS_LINUX, monkey_patch + +mark.xfail = monkey_patch IS_MAC = IS_MAC_X86 or IS_MAC_ARM diff --git a/test/test_doc_features.py b/test/test_doc_features.py index bdecdf3e..c1fca1cc 100644 --- a/test/test_doc_features.py +++ b/test/test_doc_features.py @@ -1,6 +1,9 @@ import py, os, sys from pytest import raises, skip, mark -from .support import setup_make, ispypy, IS_WINDOWS, IS_CLANG_REPL, IS_CLING, IS_CLANG_DEBUG, IS_MAC, IS_MAC_X86, IS_MAC_ARM, IS_LINUX_ARM +from .support import setup_make, ispypy, IS_WINDOWS, IS_CLANG_REPL, IS_CLING, IS_CLANG_DEBUG, IS_MAC, IS_MAC_X86, IS_MAC_ARM, IS_LINUX_ARM, monkey_patch + +mark.xfail = monkey_patch + currpath = py.path.local(__file__).dirpath() test_dct = str(currpath.join("doc_helperDict")) diff --git a/test/test_eigen.py b/test/test_eigen.py index bb4d3cb6..eb6b5e11 100644 --- a/test/test_eigen.py +++ b/test/test_eigen.py @@ -1,6 +1,9 @@ import py, os, sys from pytest import mark, raises -from .support import setup_make, IS_CLANG_REPL, IS_CLING, IS_MAC_X86 +from .support import setup_make, IS_CLANG_REPL, IS_CLING, IS_MAC_X86, monkey_patch + +mark.xfail = monkey_patch + inc_paths = [os.path.join(os.path.sep, 'usr', 'include'), os.path.join(os.path.sep, 'usr', 'local', 'include')] diff --git a/test/test_fragile.py b/test/test_fragile.py index f795b63f..465fc6a2 100644 --- a/test/test_fragile.py +++ b/test/test_fragile.py @@ -1,6 +1,9 @@ import py, os, sys from pytest import raises, skip, mark -from .support import setup_make, ispypy, IS_LINUX, IS_WINDOWS, IS_MAC_ARM, IS_CLANG_REPL, IS_MAC +from .support import setup_make, ispypy, IS_LINUX, IS_WINDOWS, IS_MAC_ARM, IS_CLANG_REPL, IS_MAC, monkey_patch + +mark.xfail = monkey_patch + currpath = py.path.local(__file__).dirpath() diff --git a/test/test_leakcheck.py b/test/test_leakcheck.py index 837e3d73..e019986c 100644 --- a/test/test_leakcheck.py +++ b/test/test_leakcheck.py @@ -1,6 +1,9 @@ import py, os, sys from pytest import mark, skip -from .support import setup_make, pylong, pyunicode, IS_CLANG_REPL +from .support import setup_make, pylong, pyunicode, IS_CLANG_REPL, monkey_patch + +mark.xfail = monkey_patch + currpath = py.path.local(__file__).dirpath() test_dct = str(currpath.join("datatypesDict")) diff --git a/test/test_lowlevel.py b/test/test_lowlevel.py index 18443b0b..823d6500 100644 --- a/test/test_lowlevel.py +++ b/test/test_lowlevel.py @@ -1,6 +1,9 @@ import py, os, sys from pytest import raises, skip, mark -from .support import setup_make, pylong, pyunicode, IS_WINDOWS, ispypy, IS_CLANG_REPL, IS_MAC +from .support import setup_make, pylong, pyunicode, IS_WINDOWS, ispypy, IS_CLANG_REPL, IS_MAC, monkey_patch + +mark.xfail = monkey_patch + currpath = py.path.local(__file__).dirpath() test_dct = str(currpath.join("datatypesDict")) diff --git a/test/test_numba.py b/test/test_numba.py index 5077fdc4..a1634305 100644 --- a/test/test_numba.py +++ b/test/test_numba.py @@ -1,7 +1,10 @@ import py, os, sys import math, time from pytest import mark, raises -from .support import setup_make +from .support import setup_make, monkey_patch + +mark.xfail = monkey_patch + try: import numba diff --git a/test/test_operators.py b/test/test_operators.py index 9ddd488b..7944250b 100644 --- a/test/test_operators.py +++ b/test/test_operators.py @@ -1,6 +1,9 @@ import py, os, sys from pytest import raises, skip, mark -from .support import setup_make, pylong, maxvalue, IS_WINDOWS, IS_MAC, IS_CLANG_REPL, IS_CLING +from .support import setup_make, pylong, maxvalue, IS_WINDOWS, IS_MAC, IS_CLANG_REPL, IS_CLING, monkey_patch + +mark.xfail = monkey_patch + currpath = py.path.local(__file__).dirpath() test_dct = str(currpath.join("operatorsDict")) diff --git a/test/test_overloads.py b/test/test_overloads.py index 67b85ac0..ce59df9d 100644 --- a/test/test_overloads.py +++ b/test/test_overloads.py @@ -1,6 +1,9 @@ import py, os, sys from pytest import raises, skip, mark -from .support import setup_make, ispypy, IS_WINDOWS, IS_MAC, IS_MAC_ARM +from .support import setup_make, ispypy, IS_WINDOWS, IS_MAC, IS_MAC_ARM, monkey_patch + +mark.xfail = monkey_patch + currpath = py.path.local(__file__).dirpath() test_dct = str(currpath.join("overloadsDict")) diff --git a/test/test_pythonify.py b/test/test_pythonify.py index 725c43e4..e39947c1 100644 --- a/test/test_pythonify.py +++ b/test/test_pythonify.py @@ -1,6 +1,9 @@ import py, os, sys from pytest import raises, skip, mark -from .support import setup_make, pylong, ispypy, IS_CLANG_REPL, IS_CLING, IS_MAC_ARM, IS_MAC_X86, IS_MAC +from .support import setup_make, pylong, ispypy, IS_CLANG_REPL, IS_CLING, IS_MAC_ARM, IS_MAC_X86, IS_MAC, monkey_patch + +mark.xfail = monkey_patch + currpath = py.path.local(__file__).dirpath() test_dct = str(currpath.join("example01Dict")) diff --git a/test/test_pythonization.py b/test/test_pythonization.py index dc3e5f0b..0b0c2f17 100644 --- a/test/test_pythonization.py +++ b/test/test_pythonization.py @@ -1,6 +1,8 @@ import py, os, sys from pytest import raises, mark -from .support import setup_make, pylong, IS_MAC_X86, IS_MAC_ARM, IS_MAC, IS_CLANG_REPL, IS_CLING +from .support import setup_make, pylong, IS_MAC_X86, IS_MAC_ARM, IS_MAC, IS_CLANG_REPL, IS_CLING, monkey_patch + +mark.xfail = monkey_patch currpath = py.path.local(__file__).dirpath() test_dct = str(currpath.join("pythonizablesDict")) diff --git a/test/test_regression.py b/test/test_regression.py index a58b3791..f6c1b2a7 100644 --- a/test/test_regression.py +++ b/test/test_regression.py @@ -1,7 +1,8 @@ import py, os, sys from pytest import raises, skip, mark -from .support import setup_make, IS_WINDOWS, ispypy, IS_CLANG_REPL, IS_CLING, IS_CLANG_DEBUG, IS_MAC_X86, IS_MAC_ARM, IS_MAC +from .support import setup_make, IS_WINDOWS, ispypy, IS_CLANG_REPL, IS_CLING, IS_CLANG_DEBUG, IS_MAC_X86, IS_MAC_ARM, IS_MAC, monkey_patch +mark.xfail = monkey_patch class TestREGRESSION: helpout = [] diff --git a/test/test_stltypes.py b/test/test_stltypes.py index 8bad09be..43dab6e6 100644 --- a/test/test_stltypes.py +++ b/test/test_stltypes.py @@ -1,7 +1,10 @@ # -*- coding: UTF-8 -*- import py, os, sys from pytest import raises, skip, mark -from .support import setup_make, pylong, pyunicode, maxvalue, ispypy, IS_CLANG_REPL, IS_CLING, IS_CLANG_DEBUG, IS_MAC_X86, IS_MAC_ARM, IS_MAC +from .support import setup_make, pylong, pyunicode, maxvalue, ispypy, IS_CLANG_REPL, IS_CLING, IS_CLANG_DEBUG, IS_MAC_X86, IS_MAC_ARM, IS_MAC, monkey_patch + +mark.xfail = monkey_patch + currpath = py.path.local(__file__).dirpath() test_dct = str(currpath.join("stltypesDict")) diff --git a/test/test_streams.py b/test/test_streams.py index 9a2dc5ac..e7e8692b 100644 --- a/test/test_streams.py +++ b/test/test_streams.py @@ -1,6 +1,9 @@ import py, os, sys from pytest import raises, mark -from .support import setup_make, IS_MAC, IS_CLANG_REPL +from .support import setup_make, IS_MAC, IS_CLANG_REPL, monkey_patch + +mark.xfail = monkey_patch + currpath = py.path.local(__file__).dirpath() test_dct = str(currpath.join("std_streamsDict")) diff --git a/test/test_templates.py b/test/test_templates.py index 0a418e27..c0f0b866 100644 --- a/test/test_templates.py +++ b/test/test_templates.py @@ -1,6 +1,9 @@ import py, os from pytest import raises, mark -from .support import setup_make, pylong, IS_CLANG_REPL, IS_CLING, IS_CLANG_DEBUG, IS_MAC_X86, IS_MAC_ARM, IS_MAC, IS_LINUX +from .support import setup_make, pylong, IS_CLANG_REPL, IS_CLING, IS_CLANG_DEBUG, IS_MAC_X86, IS_MAC_ARM, IS_MAC, IS_LINUX, monkey_patch + +mark.xfail = monkey_patch + currpath = py.path.local(__file__).dirpath() test_dct = str(currpath.join("templatesDict"))