stricter test conditions

nbconvert_a11y/axe/async_axe.py

@@ -13,20 +13,19 @@
 from subprocess import PIPE, check_output
 from playwright.async_api import Page
 
-from .base_axe_exceptions import AxeExceptions
+from nbconvert_a11y.axe.types import AxeOptions
 
-OUTER_HTML = "(node) => node.outerHTML"
-HTML_OUTER_HTML = """document.querySelector("html").outerHTML"""
-CHECK_FOR_AXE = """window.hasOwnProperty("axe")"""
-RUN_AXE = """window.axe.run({}, {})"""
-VNU_TEST = "vnu --format json -"
+from .base_axe_exceptions import AxeExceptions
 
 
 class JS:
     OUTER_HTML = "(node) => node.outerHTML"
     HTML_OUTER_HTML = """document.querySelector("html").outerHTML"""
     CHECK_FOR_AXE = """window.hasOwnProperty("axe")"""
     RUN_AXE = """window.axe.run({}, {})"""
+
+
+class SH:
     VNU_TEST = "vnu --format json -"
 
 
@@ -39,9 +38,14 @@ def get_axe() -> str:
 
 async def pw_axe(page: Page, selector=None, **config):
     # we should be able to type output
-    if not (await page.evaluate(CHECK_FOR_AXE)):
+    if not (await page.evaluate(JS.CHECK_FOR_AXE)):
         await page.evaluate(get_axe())
-    return await page.evaluate(RUN_AXE.format(selector or "document", dumps(config)))
+    return await page.evaluate(
+        JS.RUN_AXE.format(
+            selector or "document",
+            dumps(AxeOptions(**config).dict()),
+        )
+    )
 
 
 async def pw_test_axe(page: Page, selector=None, **config):
@@ -50,7 +54,7 @@ async def pw_test_axe(page: Page, selector=None, **config):
 
 async def validate_html(html: str) -> dict:
     # we should be able to type output
-    process = await create_subprocess_shell(VNU_TEST, stdin=PIPE, stdout=PIPE, stderr=PIPE)
+    process = await create_subprocess_shell(SH.VNU_TEST, stdin=PIPE, stdout=PIPE, stderr=PIPE)
     _, stderr = await process.communicate(html.encode())
     return loads(stderr)
 
@@ -59,25 +63,8 @@ async def pw_validate_html(page):
     return await validate_html(await page.outer_html())
 
 
-async def pw_screenshots(page, *selectors):
-    from IPython.display import Image
-
-    shots = {}
-    for selector in selectors:
-        shots.setdefault(selector, [])
-        async for shot in _pw_selected(page.locator(selector)):
-            shots[selector].append(Image(data=shot))
-    return shots
-
-
-async def _pw_selected(selected):
-    for nth in range(await selected.count()):
-        select = selected.nth(nth)
-        yield await select.screenshot()
-
-
 async def pw_outer_html(page: Page):
-    return await page.evaluate(HTML_OUTER_HTML)
+    return await page.evaluate(JS.HTML_OUTER_HTML)
 
 
 async def pw_accessibility_tree(page: Page):
@@ -87,6 +74,5 @@ async def pw_accessibility_tree(page: Page):
 Page.vnu = pw_validate_html
 Page.axe = pw_axe
 Page.test_axe = pw_test_axe
-Page.screenshots = pw_screenshots
 Page.outer_html = pw_outer_html
 Page.aom = pw_accessibility_tree

nbconvert_a11y/axe/pytest_axe.py

@@ -25,77 +25,10 @@
 
 from pytest import fixture
 
-from .async_axe import CHECK_FOR_AXE, RUN_AXE
+from .async_axe import JS
 from .base_axe_exceptions import AxeExceptions
 from ..exceptions import Violation, Violations
-
-
-# the default test tags start with the most strict conditions.
-# end-users can refine the TEST_TAGS they choose in their axe configuration.
-# https://github.com/dequelabs/axe-core/blob/develop/doc/API.md#axe-core-tags
-TEST_TAGS = [
-    "ACT",
-    "best-practice",
-    "experimental",
-    "wcag2a",
-    "wcag2aa",
-    "wcag2aaa",
-    "wcag21a",
-    "wcag21aa",
-    "wcag22aa",
-    "TTv5",
-]
-
-
-class Base:
-    """base class for exceptions and models"""
-
-    def __init_subclass__(cls) -> None:
-        dataclasses.dataclass(cls)
-
-    def dict(self):
-        return {k: v for k, v in dataclasses.asdict(self).items() if v is not None}
-
-    def dump(self):
-        return dumps(self.dict())
-
-
-# axe configuration should be a fixture.
-# https://github.com/dequelabs/axe-core/blob/develop/doc/API.md#api-name-axeconfigure
-class AxeConfigure(Base):
-    """axe configuration model"""
-
-    branding: str = None
-    reporter: str = None
-    checks: list = None
-    rules: list = None
-    standards: list = None
-    disableOtherRules: bool = None
-    local: str = None
-    axeVersion: str = None
-    noHtml: bool = False
-    allowedOrigins: list = dataclasses.field(default_factory=["<same_origin>"].copy)
-
-
-# axe options should be a fixture
-# https://github.com/dequelabs/axe-core/blob/develop/doc/API.md#options-parameter
-class AxeOptions(Base):
-    """axe options model"""
-
-    runOnly: list = dataclasses.field(default_factory=TEST_TAGS.copy)
-    rules: list = None
-    reporter: str = None
-    resultTypes: Any = None
-    selectors: bool = None
-    ancestry: bool = None
-    xpath: bool = None
-    absolutePaths: bool = None
-    iframes: bool = True
-    elementRef: bool = None
-    frameWaitTime: int = None
-    preload: bool = None
-    performanceTimer: bool = None
-    pingWaitTime: int = None
+from .types import AxeConfigure, AxeOptions, Base
 
 
 @dataclasses.dataclass
@@ -229,68 +162,6 @@ def raises(self, xfail=None):
                 raise exception
 
 
-@dataclasses.dataclass
-class AsyncAxe(Axe, AsyncExitStack):
-    """an axe collector that works with async playwright
-
-    we need to run playwright in sync mode for tests, but
-    we can't run playwright in sync mode in a notebook because
-    it is executed in an event loop. this class adds
-    compatability for async playwright usage for debugging.
-    """
-
-    playwright: Any = None
-    browser: Any = None
-    snapshots: list = None
-
-    def __post_init__(self):
-        AsyncExitStack.__init__(self)
-
-    async def configure(self, **config):
-        await self.page.evaluate(f"window.axe.configure({AxeConfigure(**config).dump()})")
-        self.configured = True
-        return self
-
-    async def __aenter__(self):
-        import playwright.async_api
-
-        self.playwright = await self.enter_async_context(playwright.async_api.async_playwright())
-        self.browser = await self.playwright.chromium.launch()
-        self.page = await self.browser.new_page()
-        await self.setup()
-
-        return self
-
-    async def __aexit__(self, *e):
-        await self.browser.close()
-
-    async def setup(self):
-        url = self.url
-        if isinstance(url, Path):
-            url = url.absolute().as_uri()
-        await self.page.goto(url)
-        await self.page.evaluate(get_axe())
-        return self
-
-    async def run(self, test=None, options=None, wait=None):
-        if not self.configured:
-            await self.configure()
-        if wait is not None:
-            await asyncio.sleep(wait)
-        self.results = AxeResults(
-            await self.page.evaluate(
-                f"""window.axe.run({test and dumps(test) or "document"}, {AxeOptions(**options or {}).dump()})"""
-            )
-        )
-        return self
-
-    async def screenshot(self, *args, **kwargs):
-        if not self.snapshots:
-            self.snapshots = []
-        self.snapshots.append(await self.page.screenshot(*args, **kwargs))
-        return self.snapshots[-1]
-
-
 class AxeResults(Results):
     def exception(self):
         return AxeViolations.from_violations(self.data)
@@ -322,14 +193,25 @@ def get_npm_directory(package, data=False):
     return Path(info.get("dependencies").get(package).get("path"))
 
 
-def pw_axe(page, selector=None, **config):
-    if not page.evaluate(CHECK_FOR_AXE):
+def pw_axe(page, include=None, exclude=None, **config):
+    if not page.evaluate(JS.CHECK_FOR_AXE):
         page.evaluate(get_axe())
-    return page.evaluate(RUN_AXE.format(selector and dumps(selector) or "document", dumps(config)))
+
+    if include:
+        if exclude:
+            ctx = dumps(dict(include=include, exclude=exclude))
+        else:
+            ctx = dumps(include)
+    elif exclude:
+        ctx = dumps(dict(exclude=exclude))
+    else:
+        ctx = "document"
+
+    return page.evaluate(JS.RUN_AXE.format(ctx, dumps(AxeOptions(**config).dict())))
 
 
-def pw_test_axe(page, selector=None, **config):
-    return AxeExceptions.from_test(pw_axe(page, selector, **config))
+def pw_test_axe(page, include=None, **config):
+    return AxeExceptions.from_test(pw_axe(page, include, **config))
 
 
 # attach new attributes to the synchronous page

nbconvert_a11y/axe/types.py

@@ -0,0 +1,73 @@
+
+# the default test tags start with the most strict conditions.
+# end-users can refine the TEST_TAGS they choose in their axe configuration.
+# https://github.com/dequelabs/axe-core/blob/develop/doc/API.md#axe-core-tags
+import dataclasses
+from json import dumps
+from typing import Any
+
+
+TEST_TAGS = [
+    "ACT",
+    "best-practice",
+    "experimental",
+    "wcag2a",
+    "wcag2aa",
+    "wcag2aaa",
+    "wcag21a",
+    "wcag21aa",
+    "wcag22aa",
+    "TTv5",
+]
+
+
+class Base:
+    """base class for exceptions and models"""
+
+    def __init_subclass__(cls) -> None:
+        dataclasses.dataclass(cls)
+
+    def dict(self):
+        return {k: v for k, v in dataclasses.asdict(self).items() if v is not None}
+
+    def dump(self):
+        return dumps(self.dict())
+
+
+# axe configuration should be a fixture.
+# https://github.com/dequelabs/axe-core/blob/develop/doc/API.md#api-name-axeconfigure
+class AxeConfigure(Base):
+    """axe configuration model"""
+
+    branding: str = None
+    reporter: str = None
+    checks: list = None
+    rules: list = None
+    standards: list = None
+    disableOtherRules: bool = None
+    local: str = None
+    axeVersion: str = None
+    noHtml: bool = False
+    allowedOrigins: list = dataclasses.field(default_factory=["<same_origin>"].copy)
+
+
+# axe options should be a fixture
+# https://github.com/dequelabs/axe-core/blob/develop/doc/API.md#options-parameter
+class AxeOptions(Base):
+    """axe options model"""
+
+    runOnly: list = dataclasses.field(default_factory=TEST_TAGS.copy)
+    rules: list = None
+    reporter: str = None
+    resultTypes: Any = None
+    selectors: bool = None
+    ancestry: bool = None
+    xpath: bool = None
+    absolutePaths: bool = None
+    iframes: bool = True
+    elementRef: bool = None
+    frameWaitTime: int = None
+    preload: bool = None
+    performanceTimer: bool = None
+    pingWaitTime: int = None
+

nbconvert_a11y/pytest_w3c.py

@@ -21,7 +21,7 @@
 import pytest
 import requests
 
-from nbconvert_a11y.pytest_axe import Collector, Results, Violation
+from nbconvert_a11y.axe.pytest_axe import Collector, Results, Violation
 
 HERE = Path(__file__).parent
 

tests/test_color_themes.py

@@ -1,6 +1,7 @@
 from time import sleep
 from pytest import fixture
 
+from nbconvert_a11y.axe.axe_exceptions import color_contrast_enhanced
 from nbconvert_a11y.exporter import THEMES
 from tests.conftest import CONFIGURATIONS, NOTEBOOKS
 from playwright.sync_api import expect
@@ -25,7 +26,7 @@ def test_dark_themes(lorenz):
     _test_no_textarea(lorenz)
     # verify the themes are consistent
     assert lorenz.locator(f"#nb-light-highlight").get_attribute("media") == "not screen"
-    assert lorenz.test_axe(dict(include=[".nb-source"])).xfail()
+    assert lorenz.test_axe(dict(include=[".nb-source"])).xfail(color_contrast_enhanced)
     # accessible pygments disclsoes that we should expect some color contrast failures on some themes.
     # there isnt much code which might not generate enough conditions to create color contrast issues.