workflow,pytest: add new --error-on-skip for pytest

To ensure we do not skip tests in GH actions that we want to run
this commit add a new explicit way to allowlist tests that are okay
to skip.

This is done via a small pytest plugin and the GH action will
now use `-k "not test_selinux_ro"` because this selinux tests
requires a real selinux which is not available on GH runners.

.github/workflows/test.yml

@@ -37,6 +37,9 @@ jobs:
           # it to vfs for the local storage skopeo stage test.
           sed -i 's/overlay/vfs/g' /usr/share/containers/storage.conf  # default system config
           sed -i 's/overlay/vfs/g' /etc/containers/storage.conf || true  # potential overrides
+
+          # not running "test_selinux_ro" here as it needs a real selinux
+          TEST_SKIP="not test_selinux_ro"
           if [ "${{ matrix.test }}" == "parallel" ]; then
               # 4 is a bit arbitrary
               TEST_WORKERS="-n 4"
@@ -55,7 +58,7 @@ jobs:
               fi
           fi
           OSBUILD_TEST_STORE="${{ env.OSBUILD_TEST_STORE }}" \
-          tox -e "${{ matrix.environment }}" $TOX_ARGS -- -rs $TEST_WORKERS -k "$TEST_CATEGORY"
+          tox -e "${{ matrix.environment }}" $TOX_ARGS -- -rs $TEST_WORKERS -k "$TEST_CATEGORY" --error-on-skip -k "$TEST_SKIP"
 
   v1_manifests:
     name: "Assembler test (legacy)"

conftest.py

@@ -0,0 +1,20 @@
+import pytest
+
+
+@pytest.hookimpl(hookwrapper=True)
+def pytest_runtest_makereport(item, call):
+    outcome = yield
+    res = outcome.get_result()
+    if res.skipped:
+        error_on_skip = item.config.getoption('--error-on-skip')
+
+        reason = str(call.excinfo.value)
+        if error_on_skip:
+            res.outcome = "failed"
+            res.longrepr = f"skipping not allowed: {res.longrepr}"
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        '--error-on-skip', action="store_true",
+    )