update tests

nbconvert_a11y/templates/a11y/components/core.html.j2

@@ -33,7 +33,7 @@ at some point effort should be put into separating context dependent and indepen
 <label id="nb-{{name}}-select">{{title}}
     <select name="{{name}}">
         {% for k, v in values.items() %}
-        <option value="{{v}}" {% if k==default %} selected{% endif %}>{{k}}</option>
+        <option value="{{v}}" {% if v==default %} selected{% endif %}>{{k}}</option>
         {% endfor %}
         {% for k, v in (disabled or {}).items() %}
         <option value="{{v}}" disabled>{{k}}</option>

tests/conftest.py

@@ -8,7 +8,6 @@
 from flask import Flask, request, url_for
 
 import nbconvert.nbconvertapp
-from nbconvert import get_exporter
 
 collect_ignore = ["notebooks/lorenz.ipynb"]
 HERE = Path(__file__).parent
@@ -18,6 +17,7 @@
 HTML = EXPORTS / "html"
 CI = environ.get("CI")
 
+
 TestLoader.testMethodPrefix = "test", "xfail"
 
 
@@ -72,12 +72,12 @@ def app():
     return make_app()
 
 
-@pytest.fixture(scope="session")
-def notebook(live_server):
-    def url(exporter="a11y", name="lorenz-executed.ipynb", **qs):
-        return url_for("nb", exporter=exporter, name=name, **qs, _external=True)
+# @pytest.fixture(scope="session")
+# def notebook(live_server):
+#     def url(exporter="a11y", name="lorenz-executed.ipynb", **qs):
+#         return url_for("nb", exporter=exporter, name=name, **qs, _external=True)
 
-    return url
+#     return url
 
 
 if __name__ == "__main__":

tests/test_a11y_baseline.py

@@ -4,33 +4,22 @@
 * test the accessibility of nbconvert-a11y dialogs
 """
 
-
 from pytest import mark
 
-from nbconvert_a11y.pytest_axe import JUPYTER_WIDGETS, MATHJAX, SA11Y, AxeViolation
+from nbconvert_a11y.test_utils import SELECTORS as S
+from conftest import NOTEBOOKS, CONFIGURATIONS
 
 
-@mark.parametrize(
-    "config,exporter_name,name",
-    [
-        ("a11y.py", "a11y-table", "lorenz-executed.ipynb"),
-        ("section.py", "a11y-landmark", "lorenz-executed.ipynb"),
-        ("list.py", "a11y-list", "lorenz-executed.ipynb"),
-    ],
-)
-def test_axe(axe, notebook, config, exporter_name, name):
+@mark.parametrize("config", ["a11y.py", "section.py", "list.py"])
+@mark.parametrize("nb", ["lorenz-executed.ipynb"])
+def test_axe(page, config, nb):
     """Verify the baseline templates satisify all rules update AAA.
 
     any modifications to the template can only degrade accessibility.
     this baseline is critical for adding more features. all testing piles
     up without automation. these surface protections allow more manual testing
     or verified conformations of html.
     """
-    test = axe(notebook(exporter_name, name, config=config))
-    # ignore mathjax at the moment. we might be able to turne mathjax to have better
-    # accessibility. https://github.com/Iota-School/notebooks-for-all/issues/81
-    test.run({"exclude": [JUPYTER_WIDGETS, MATHJAX, SA11Y]})
-    try:
-        test.raises()
-    except* AxeViolation["serious-color-contrast-enhanced"]:
-        ...
+
+    page.from_notebook(NOTEBOOKS / nb, CONFIGURATIONS / config)
+    assert page.test_axe(exclude=S.THIRD_PARTY).xfail()

tests/test_a11y_settings.py

@@ -1,18 +1,18 @@
 """specific ui and accessibility tests for the custom a11y template."""
 
-
 from pytest import fixture, mark, param
 
-from nbconvert_a11y.pytest_axe import AxeViolation, Axe
-from tests.test_smoke import CONFIGURATIONS, NOTEBOOKS, get_target_html
+from nbconvert_a11y.axe.base_axe_exceptions import AxeExceptions
+from tests.test_smoke import CONFIGURATIONS, NOTEBOOKS
+import nbconvert_a11y.test_utils
 
-NEEDS_WORK = mark.xfail(reason="state needs work", raises=Violations)
+NEEDS_WORK = mark.xfail(reason="state needs work", raises=AxeExceptions)
 
 
 @fixture()
-def lorenz(page, notebook):
-    axe = Axe(page=page, url=notebook("a11y", "lorenz-executed.ipynb", config="a11y.py"))
-    return axe.configure()
+def lorenz(page):
+    page.from_notebook(NOTEBOOKS / "lorenz-executed.ipynb", CONFIGURATIONS / "a11y.py")
+    return page
 
 
 class JS:
@@ -26,29 +26,20 @@ class JS:
     [
         "[aria-controls=nb-settings]",
         "[aria-controls=nb-help]",
-        # "[aria-controls=nb-metadata]",
-        param("[aria-controls=nb-expanded-dialog]", marks=mark.xfail(reason=NEEDS_WORK)),
         param("[aria-controls=nb-visibility-dialog]", marks=mark.xfail(reason=NEEDS_WORK)),
     ],
 )
 def test_dialogs(lorenz, dialog):
     """Test the controls in dialogs."""
     # dialogs are not tested in the baseline axe test. they need to be active to test.
     # these tests activate the dialogs to assess their accessibility with the active dialogs.
-    lorenz.page.click(dialog)
-    test = lorenz.run()
-    try:
-        test.raises()
-    except* AxeViolation["serious-color-contrast-enhanced"]:
-        ...
-    except* AxeViolation["serious-target-size"]:
-        "the line height raises a target size error for some reason"
+    lorenz.click(dialog)
+    assert lorenz.test_axe().xfail()
+
 
 def test_settings_font_size(lorenz):
     """Test that the settings make their expected changes."""
-    assert (
-        lorenz.page.evaluate(JS.SNIPPET_FONT_SIZE) == "16px"
-    ), "the default font size is unexpected"
-    lorenz.page.click("[aria-controls=nb-settings]")
-    lorenz.page.locator("[name=font-size]").select_option("xx-large")
-    assert lorenz.page.evaluate(JS.SNIPPET_FONT_SIZE) == "32px", "font size not changed"
+    assert lorenz.evaluate(JS.SNIPPET_FONT_SIZE) == "16px", "the default font size is unexpected"
+    lorenz.click("[aria-controls=nb-settings]")
+    lorenz.locator("[name=font-size]").select_option("xx-large")
+    assert lorenz.evaluate(JS.SNIPPET_FONT_SIZE) == "32px", "font size not changed"

tests/test_color_themes.py

@@ -1,31 +1,41 @@
+from time import sleep
 from pytest import fixture
 
 from nbconvert_a11y.exporter import THEMES
-from nbconvert_a11y.pytest_axe import Axe
+from tests.conftest import CONFIGURATIONS, NOTEBOOKS
+from playwright.sync_api import expect
+import nbconvert_a11y.test_utils
 
 LORENZ = "lorenz-executed.ipynb"
 
 
 @fixture(params=list(THEMES))
-def lorenz(page, request, notebook):
-    return Axe(
-        page=page,
-        url=notebook(
-            "a11y", LORENZ, color_theme=request.param, include_settings=True, wcag_priority="AA"
-        ),
-    ).configure()
+def lorenz(page, request):
+    nb, config = NOTEBOOKS / LORENZ, CONFIGURATIONS / "a11y.py"
+    page.from_notebook(
+        nb, config, color_theme=request.param, include_settings=True, wcag_priority="AA"
+    )
+    return page
 
 
 def test_dark_themes(lorenz):
-    lorenz.page.click("""[aria-controls="nb-settings"]""")
-    lorenz.page.locator("select[name=color-scheme]").select_option("dark mode")
-    lorenz.page.keyboard.press("Escape")
-    # force the background to be black because axe detects a white background in dark mode.
-    lorenz.page.eval_on_selector("body", """x => x.style.backgroundColor = `#000000`""")
-    lorenz.run(dict(include=[".nb-source"])).raises()
+    lorenz.click("""[aria-controls="nb-settings"]""")
+    lorenz.locator("select[name=color-scheme]").select_option("dark mode")
+    lorenz.keyboard.press("Escape")
+    _test_no_textarea(lorenz)
+    # verify the themes are consistent
+    assert lorenz.locator(f"#nb-light-highlight").get_attribute("media") == "not screen"
+    assert lorenz.test_axe(dict(include=[".nb-source"])).xfail()
     # accessible pygments disclsoes that we should expect some color contrast failures on some themes.
     # there isnt much code which might not generate enough conditions to create color contrast issues.
 
 
 def test_light_themes(lorenz):
-    lorenz.run(dict(include=[".nb-source"])).raises()
+    _test_no_textarea(lorenz)
+    assert lorenz.locator(f"#nb-dark-highlight").get_attribute("media") == "not screen"
+    assert lorenz.test_axe(dict(include=[".nb-source"])).xfail()
+
+
+def _test_no_textarea(page):
+    for element in page.locator("textarea.nb-source").all():
+        expect(element).not_to_be_visible()