Merge pull request #3870 from Zac-HD/pyodide-in-ci

Test against Pyodide / Emscripten in CI

.github/workflows/main.yml

@@ -192,6 +192,55 @@ jobs:
     - name: Run tests
       run: TASK=${{ matrix.task }} ./build.sh
 
+  # See https://pyodide.org/en/stable/development/building-and-testing-packages.html#testing-packages-against-pyodide
+  # and https://github.com/numpy/numpy/blob/9a650391651c8486d8cb8b27b0e75aed5d36033e/.github/workflows/emscripten.yml
+  test-pyodide:
+    runs-on: ubuntu-latest
+    env:
+      NODE_VERSION: 18
+      # Note that the versions below must be updated in sync; we've automated
+      # that with `update_pyodide_versions()` in our weekly cronjob.
+      PYODIDE_VERSION: 0.25.0
+      PYTHON_VERSION: 3.11.3
+      EMSCRIPTEN_VERSION: 3.1.46
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+      - name: Set up Node
+        uses: actions/setup-node@b39b52d1213e96004bfcb1c61a8a6fa8ab84f3e8 # v4.0.1
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+      - name: Set up Emscripten
+        uses: mymindstorm/setup-emsdk@6ab9eb1bda2574c4ddb79809fc9247783eaf9021 # v14
+        with:
+          version: ${{ env.EMSCRIPTEN_VERSION }}
+          actions-cache-folder: emsdk-cache
+      - name: Build
+        run: |
+          pip install "pydantic<2" pyodide-build==$PYODIDE_VERSION
+          cd hypothesis-python/
+          CFLAGS=-g2 LDFLAGS=-g2 pyodide build
+      - name: Set up Pyodide venv and install dependencies
+        run: |
+          pip install --upgrade setuptools pip wheel
+          python hypothesis-python/setup.py bdist_wheel
+          pip download --dest=dist/ hypothesis-python/ pytest tzdata  # fetch all the wheels
+
+          rm dist/packaging-*.whl  # fails with `invalid metadata entry 'name'`
+
+          pyodide venv .venv-pyodide
+          source .venv-pyodide/bin/activate
+          pip install dist/*.whl
+      - name: Run tests
+        run: |
+          source .venv-pyodide/bin/activate
+          python -m pytest hypothesis-python/tests/cover
+
   deploy:
     if: "github.event_name == 'push' && github.repository == 'HypothesisWorks/hypothesis'"
     runs-on: ubuntu-latest

.gitignore

@@ -13,6 +13,7 @@
 .runtimes
 /hypothesis-python/branch-check
 /pythonpython3.*
+.pyodide-xbuildenv
 
 # python
 

hypothesis-python/setup.py

@@ -61,9 +61,9 @@ def local_file(name):
     "dpcontracts": ["dpcontracts>=0.4"],
     "redis": ["redis>=3.0.0"],
     # zoneinfo is an odd one: every dependency is conditional, because they're
-    # only necessary on old versions of Python or Windows systems.
+    # only necessary on old versions of Python or Windows systems or emscripten.
     "zoneinfo": [
-        "tzdata>=2023.4 ; sys_platform == 'win32'",
+        "tzdata>=2023.4 ; sys_platform == 'win32' or sys_platform == 'emscripten'",
         "backports.zoneinfo>=0.2.1 ; python_version<'3.9'",
     ],
     # We only support Django versions with upstream support - see

hypothesis-python/tests/common/utils.py

@@ -47,6 +47,20 @@ def raises(expected_exception, match=None):
             ) from None
 
 
+try:
+    from pytest import mark
+except ModuleNotFoundError:
+
+    def skipif_emscripten(f):
+        return f
+
+else:
+    skipif_emscripten = mark.skipif(
+        sys.platform == "emscripten",
+        reason="threads, processes, etc. are not available in the browser",
+    )
+
+
 no_shrink = tuple(set(settings.default.phases) - {Phase.shrink, Phase.explain})
 
 

hypothesis-python/tests/cover/test_asyncio.py

@@ -17,6 +17,8 @@
 from hypothesis import assume, given, strategies as st
 from hypothesis.internal.compat import PYPY
 
+from tests.common.utils import skipif_emscripten
+
 
 def coro_decorator(f):
     with warnings.catch_warnings():
@@ -62,12 +64,24 @@ def test_foo(self, x):
 
 
 class TestAsyncioRun(TestCase):
+    # In principle, these tests could indeed run on emscripten if we grab the existing
+    # event loop and run them there.  However, that seems to have hit an infinite loop
+    # and so we're just skipping them for now and will revisit later.
+
     def execute_example(self, f):
         asyncio.run(f())
 
+    @skipif_emscripten
     @given(x=st.text())
     @coro_decorator
-    def test_foo(self, x):
+    def test_foo_yield_from(self, x):
         assume(x)
         yield from asyncio.sleep(0.001)
         assert x
+
+    @skipif_emscripten
+    @given(st.text())
+    async def test_foo_await(self, x):
+        assume(x)
+        await asyncio.sleep(0.001)
+        assert x

hypothesis-python/tests/cover/test_cache_implementation.py

@@ -25,6 +25,8 @@
 )
 from hypothesis.internal.cache import GenericCache, LRUReusedCache
 
+from tests.common.utils import skipif_emscripten
+
 
 class LRUCache(GenericCache):
     __slots__ = ("__tick",)
@@ -299,6 +301,7 @@ def test_iterates_over_remaining_keys():
     assert sorted(cache) == [1, 2]
 
 
+@skipif_emscripten
 def test_cache_is_threadsafe_issue_2433_regression():
     errors = []
 

hypothesis-python/tests/cover/test_interactive_example.py

@@ -12,7 +12,6 @@
 import warnings
 from decimal import Decimal
 
-import pexpect
 import pytest
 
 from hypothesis import example, find, given, strategies as st
@@ -25,7 +24,7 @@
 from hypothesis.internal.compat import WINDOWS
 
 from tests.common.debug import find_any
-from tests.common.utils import fails_with
+from tests.common.utils import fails_with, skipif_emscripten
 
 pytest_plugins = "pytester"
 
@@ -107,8 +106,11 @@ def test_selftests_exception_contains_note(pytester):
         assert "helper methods in tests.common.debug" in "\n".join(result.outlines)
 
 
+@skipif_emscripten
 @pytest.mark.skipif(WINDOWS, reason="pexpect.spawn not supported on Windows")
 def test_interactive_example_does_not_emit_warning():
+    import pexpect
+
     try:
         child = pexpect.spawn(f"{sys.executable} -Werror")
         child.expect(">>> ", timeout=10)

hypothesis-python/tests/cover/test_lazy_import.py

@@ -12,6 +12,8 @@
 import subprocess
 import sys
 
+from tests.common.utils import skipif_emscripten
+
 SHOULD_NOT_IMPORT_TEST_RUNNERS = """
 import sys
 import unittest
@@ -37,6 +39,7 @@ def test_does_not_import_pytest(self, x):
 """
 
 
+@skipif_emscripten
 def test_hypothesis_does_not_import_test_runners(tmp_path):
     # We obviously can't use pytest to check that pytest is not imported,
     # so for consistency we use unittest for all three non-stdlib test runners.

hypothesis-python/tests/cover/test_settings.py

@@ -38,6 +38,7 @@
     checks_deprecated_behaviour,
     counts_calls,
     fails_with,
+    skipif_emscripten,
     validate_deprecation,
 )
 
@@ -231,7 +232,7 @@ def test_settings_alone():
 
 def test_settings_alone(testdir):
     script = testdir.makepyfile(TEST_SETTINGS_ALONE)
-    result = testdir.runpytest(script)
+    result = testdir.runpytest_inprocess(script)
     out = "\n".join(result.stdout.lines)
     assert (
         "Using `@settings` on a test without `@given` is completely pointless." in out
@@ -284,6 +285,7 @@ def test_settings_as_decorator_must_be_on_callable():
 """
 
 
+@skipif_emscripten
 def test_puts_the_database_in_the_home_dir_by_default(tmp_path):
     script = tmp_path.joinpath("assertlocation.py")
     script.write_text(ASSERT_DATABASE_PATH, encoding="utf-8")

hypothesis-python/tests/cover/test_stateful.py

@@ -129,6 +129,7 @@ def action(self, d):
         raise AssertionError
 
 
+@Settings(stateful_step_count=10, max_examples=30)  # speed this up
 class MachineWithConsumingRule(RuleBasedStateMachine):
     b1 = Bundle("b1")
     b2 = Bundle("b2")
@@ -152,7 +153,7 @@ def depopulate_b1(self, consumed):
         self.consumed_counter += 1
         return consumed
 
-    @rule(consumed=lists(consumes(b1)))
+    @rule(consumed=lists(consumes(b1), max_size=3))
     def depopulate_b1_multiple(self, consumed):
         self.consumed_counter += len(consumed)
 

hypothesis-python/tests/cover/test_testdecorators.py

@@ -36,6 +36,7 @@
     fails_with,
     no_shrink,
     raises,
+    skipif_emscripten,
 )
 
 # This particular test file is run under both pytest and nose, so it can't
@@ -331,6 +332,7 @@ def test_blah(x):
         test_blah()
 
 
+@skipif_emscripten
 def test_can_run_with_database_in_thread():
     results = []
 

hypothesis-python/tests/cover/test_unittest.py

@@ -17,7 +17,7 @@
 from hypothesis import given, strategies as st
 from hypothesis.errors import FailedHealthCheck, HypothesisWarning
 
-from tests.common.utils import fails_with
+from tests.common.utils import fails_with, skipif_emscripten
 
 
 class Thing_with_a_subThing(unittest.TestCase):
@@ -65,6 +65,7 @@ def test_subtest(self, s):
 """
 
 
+@skipif_emscripten
 @pytest.mark.parametrize("err", [[], ["-Werror"]])
 def test_subTest_no_self(testdir, err):
     # https://github.com/HypothesisWorks/hypothesis/issues/2462

tooling/src/hypothesistooling/__main__.py

@@ -297,7 +297,7 @@ def update_python_versions():
     pip_tool("shed", str(thisfile))
 
     # Automatically sync ci_version with the version in build.sh
-    build_sh = pathlib.Path(tools.ROOT) / "build.sh"
+    build_sh = tools.ROOT / "build.sh"
     sh_before = build_sh.read_text(encoding="utf-8")
     sh_after = re.sub(r"3\.\d\d?\.\d\d?", best[ci_version], sh_before)
     if sh_before != sh_after:
@@ -306,6 +306,33 @@ def update_python_versions():
         build_sh.chmod(0o755)
 
 
+def update_pyodide_versions():
+    vers_re = r"(\d+\.\d+\.\d+)"
+    pyodide_version = max(
+        # Don't just pick the most recent version; find the highest stable version.
+        re.findall(
+            f"pyodide_build-{vers_re}-py3-none-any.whl",  # excludes pre-releases
+            requests.get("https://pypi.org/simple/pyodide-build/").text,
+        ),
+        key=lambda version: tuple(int(x) for x in version.split(".")),
+    )
+    makefile_url = f"https://raw.githubusercontent.com/pyodide/pyodide/{pyodide_version}/Makefile.envs"
+    python_version, emscripten_version = re.search(
+        rf"export PYVERSION \?= {vers_re}\nexport PYODIDE_EMSCRIPTEN_VERSION \?= {vers_re}\n",
+        requests.get(makefile_url).text,
+    ).groups()
+
+    ci_file = tools.ROOT / ".github/workflows/main.yml"
+    config = ci_file.read_text(encoding="utf-8")
+    for name, var in [
+        ("PYODIDE", pyodide_version),
+        ("PYTHON", python_version),
+        ("EMSCRIPTEN", emscripten_version),
+    ]:
+        config = re.sub(f"{name}_VERSION: {vers_re}", f"{name}_VERSION: {var}", config)
+    ci_file.write_text(config, encoding="utf-8")
+
+
 def update_vendored_files():
     vendor = pathlib.Path(hp.PYTHON_SRC) / "hypothesis" / "vendor"
 
@@ -317,7 +344,8 @@ def update_vendored_files():
     if fname.read_bytes().splitlines()[1:] != new.splitlines()[1:]:
         fname.write_bytes(new)
 
-    # Always require the latest version of the tzdata package
+    # Always require the most recent version of tzdata - we don't need to worry about
+    # pre-releases because tzdata is a 'latest data' package  (unlike pyodide-build).
     tz_url = "https://pypi.org/pypi/tzdata/json"
     tzdata_version = requests.get(tz_url).json()["info"]["version"]
     setup = pathlib.Path(hp.BASE_DIR, "setup.py")
@@ -344,6 +372,7 @@ def upgrade_requirements():
         with open(hp.RELEASE_FILE, mode="w", encoding="utf-8") as f:
             f.write(f"RELEASE_TYPE: patch\n\n{msg}")
     update_python_versions()
+    update_pyodide_versions()
     subprocess.call(["git", "add", "."], cwd=tools.ROOT)