[WIP] CI: Added tests on FreeBSD, OpenBSD & NetBSD

.github/codecov.yml

@@ -1,7 +1,13 @@
 comment:
   require_changes: false  # Always show messages if codecov is triggered
   layout: "flags"  # Only show coverage by flags
-  after_n_builds: 18  # nb_tested_python_version(==3) * (nb_functional_test_runs(==3) + nb_unit_test_runs(==3))
+  # unit and functional tests run:
+  # Python 3.11: Linux(==1), Windows(==1), MacOS(==1), FreeBSD(==1).
+  # Python 3.12: Linux(==1), Windows(==1), MacOS(==1).
+  # Python 3.13: Linux(==1), Windows(==1), MacOS(==1).
+  # Final calculation:
+  # (nb_builds_per_python_version(==4 + 3 + 3) * (nb_test_runs (unit + functional) (==2))
+  after_n_builds: 20
 coverage:
   status:
     project:

.github/workflows/build.yml

@@ -5,6 +5,8 @@ on:
     types:
       - opened
       - synchronize
+      - converted_to_draft
+      - ready_for_review
     branches:
       - main
     paths-ignore:
@@ -59,7 +61,9 @@ concurrency:
 
 jobs:
   build:
-    if: github.event_name != 'push' || !startsWith(github.event.head_commit.message, 'Bump version:')
+    if: |
+      (github.event_name != 'push' || !startsWith(github.event.head_commit.message, 'Bump version:'))
+      && (github.event_name != 'pull_request' || github.event.pull_request.draft != true)
     runs-on: ubuntu-24.04
     env:
       SOURCE_DATE_EPOCH: ${{ inputs.SOURCE_DATE_EPOCH }}

.github/workflows/lint.yml

@@ -6,6 +6,8 @@ on:
     types:
       - opened
       - synchronize
+      - converted_to_draft
+      - ready_for_review
     branches:
       - main
     paths:
@@ -38,7 +40,9 @@ concurrency:
 
 jobs:
   type-hinting:
-    if: github.event_name != 'push' || !startsWith(github.event.head_commit.message, 'Bump version:')
+    if: |
+      (github.event_name != 'push' || !startsWith(github.event.head_commit.message, 'Bump version:'))
+      && (github.event_name != 'pull_request' || github.event.pull_request.draft != true)
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false

.github/workflows/test.yml

@@ -12,6 +12,8 @@ on:
     types:
       - opened
       - synchronize
+      - converted_to_draft
+      - ready_for_review
     branches:
       - main
     paths:
@@ -44,6 +46,10 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
   cancel-in-progress: true
 
+env:
+  PYTEST_VERBOSE_FLAG: ${{ inputs.verbose && '-v' || '' }}
+  PDM_VERSION: '2.19.3'  # Used by FreeBSD VM
+
 jobs:
   tests:
     if: |
@@ -62,8 +68,6 @@ jobs:
             tox_py: py312
           - python_version: '3.13'
             tox_py: py313
-    env:
-      PYTEST_VERBOSE_FLAG: ${{ inputs.verbose && '-v' || '' }}
 
     name: test (${{ matrix.os }}, ${{ matrix.python_version }})
     steps:
@@ -108,3 +112,77 @@ jobs:
             test-functional,
             OS-${{ runner.os }},
             Py-${{ matrix.python_version }}
+
+  test-freebsd:
+    if: |
+      (github.event_name != 'push' || !startsWith(github.event.head_commit.message, 'Bump version:'))
+      && (github.event_name != 'pull_request' || (github.event.pull_request.draft != true && !contains(github.event.pull_request.labels.*.name, 'pr-skip-test')))
+    runs-on: ubuntu-24.04
+    strategy:
+      fail-fast: false
+      matrix:
+        # TODO: Add test with other python version
+        # c.f. https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=271673
+        include:
+          # py3XX-sqlite3 is needed for coverage.py
+          - python_version: '3.11'
+            freebsd_ports: >-
+              python311
+              py311-sqlite3
+            tox_py: py311
+
+    name: test (FreeBSD, ${{ matrix.python_version }})
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Launch tests
+        # Add 5 minutes to let the VM boot and install dependencies
+        timeout-minutes: 25
+        uses: vmactions/freebsd-vm@v1
+        with:
+          release: '14.1'
+          usesh: true
+          prepare: |
+            set -e
+            pkg install -y curl git ${{ matrix.freebsd_ports }}
+            curl -sSL https://pdm-project.org/install-pdm.py | python${{ matrix.python_version }} - --version=${{ env.PDM_VERSION }} --path=/usr/local
+            pdm config check_update false
+            pdm config install.cache true
+          run: |
+            pdm install --frozen-lockfile --global --project=. --no-self --no-default --dev --group=tox
+            tox --version
+            tox run -f ${{ matrix.tox_py }} -- ${{ env.PYTEST_VERBOSE_FLAG }}
+            tox run -f coverage
+            rm -rf .tox
+            find . -name '__pycache__' | xargs rm -rf
+      - name: Check files in workspace
+        if: always()
+        run: ls -lA
+      # Currently, it is not possible to send several files with per-file tags.
+      # This is why the step is copy-paste twice.
+      # Issue: https://github.com/codecov/codecov-action/issues/1522
+      - name: Upload (unit tests) coverage to codecov
+        if: hashFiles('coverage.unit.xml') != ''  # Rudimentary `file.exists()`
+        uses: codecov/codecov-action@v4
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          disable_search: true
+          files: >-
+            coverage.unit.xml
+          flags: >-
+            test-unit,
+            OS-FreeBSD,
+            Py-${{ matrix.python_version }}
+      - name: Upload (functional tests) coverage to codecov
+        if: hashFiles('coverage.functional.xml') != ''  # Rudimentary `file.exists()`
+        uses: codecov/codecov-action@v4
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          disable_search: true
+          files: >-
+            coverage.functional.xml
+          flags: >-
+            test-functional,
+            OS-FreeBSD,
+            Py-${{ matrix.python_version }}

pyproject.toml

@@ -106,11 +106,13 @@ test = [
     "pytest-cov~=5.0",
     "pytest-asyncio~=0.24.0",
     "trove-classifiers>=2023.11.9",
-    "trustme~=1.0",
     # "pytest-retry~=1.6",
     # Temporary use VCS to get the modifications added on main (c.f. https://github.com/str0zzapreti/pytest-retry/pull/39)
     "pytest-retry @ git+https://github.com/str0zzapreti/pytest-retry.git@bb465fff6f01f3f90a77229468f7e08a3bdbce20",
 ]
+test-ssl = [
+    "trustme~=1.0",
+]
 test-trio = [
     "pytest-trio~=0.8.0",
 ]

tests/functional_test/test_communication/conftest.py

@@ -5,15 +5,18 @@
 from contextlib import ExitStack
 from functools import partial
 from socket import AF_INET, AF_INET6, SOCK_DGRAM, SOCK_STREAM, has_ipv6 as HAS_IPV6, socket as Socket
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 from easynetwork.protocol import AnyStreamProtocolType, BufferedStreamProtocol, DatagramProtocol, StreamProtocol
 
 import pytest
-import trustme
 
 from .serializer import BadSerializeStringSerializer, NotGoodStringSerializer, StringSerializer
 
+if TYPE_CHECKING:
+    import trustme
+
+
 _FAMILY_TO_LOCALHOST: dict[int, str] = {
     AF_INET: "127.0.0.1",
     AF_INET6: "::1",
@@ -123,6 +126,11 @@ def socket_pair(localhost_ip: str, tcp_socket_factory: Callable[[], Socket]) ->
 
 @pytest.fixture(scope="session")
 def ssl_certificate_authority() -> trustme.CA:
+    try:
+        import trustme
+    except ModuleNotFoundError:
+        pytest.skip("trustme is not installed")
+
     return trustme.CA()
 
 

tests/functional_test/test_communication/test_async/test_client/test_udp.py

@@ -87,7 +87,7 @@ async def test____send_packet____default(self, client: AsyncUDPNetworkClient[str
         async with asyncio.timeout(3):
             assert await server.recvfrom() == (b"ABCDEF", client.get_local_address())
 
-    @PlatformMarkers.runs_only_on_platform("linux", "Windows and MacOS do not raise error")
+    @PlatformMarkers.runs_only_on_platform("linux", "Windows, MacOS and BSD-like do not raise error")
     async def test____send_packet____connection_refused(
         self,
         client: AsyncUDPNetworkClient[str, str],
@@ -97,7 +97,7 @@ async def test____send_packet____connection_refused(
         with pytest.raises(ConnectionRefusedError):
             await client.send_packet("ABCDEF")
 
-    @PlatformMarkers.runs_only_on_platform("linux", "Windows and MacOS do not raise error")
+    @PlatformMarkers.runs_only_on_platform("linux", "Windows, MacOS and BSD-like do not raise error")
     async def test____send_packet____connection_refused____after_previous_successful_try(
         self,
         client: AsyncUDPNetworkClient[str, str],

tests/functional_test/test_communication/test_async/test_server/test_tcp.py

@@ -638,6 +638,7 @@ async def test____serve_forever____accept_client____client_sent_RST_packet_right
         await asyncio.sleep(0.1)
 
         # On Linux: ENOTCONN error should not create a big Traceback error
+        # On BSD: ECONNABORTED error on accept() should not create a big Traceback error
         assert len(caplog.records) == 0
 
     async def test____serve_forever____client_extra_attributes(

tests/functional_test/test_communication/test_sync/test_client/test_udp.py

@@ -58,13 +58,13 @@ def test____send_packet____default(self, client: UDPNetworkClient[str, str], ser
         client.send_packet("ABCDEF")
         assert server.recvfrom(1024) == (b"ABCDEF", client.get_local_address())
 
-    @PlatformMarkers.runs_only_on_platform("linux", "Windows and MacOS do not raise error")
+    @PlatformMarkers.runs_only_on_platform("linux", "Windows, MacOS and BSD-like do not raise error")
     def test____send_packet____connection_refused(self, client: UDPNetworkClient[str, str], server: Socket) -> None:
         server.close()
         with pytest.raises(ConnectionRefusedError):
             client.send_packet("ABCDEF")
 
-    @PlatformMarkers.runs_only_on_platform("linux", "Windows and MacOS do not raise error")
+    @PlatformMarkers.runs_only_on_platform("linux", "Windows, MacOS and BSD-like do not raise error")
     def test____send_packet____connection_refused____after_previous_successful_try(
         self,
         client: UDPNetworkClient[str, str],

tests/tools.py

@@ -52,9 +52,14 @@ def skipif_platform_macOS_because(reason: str, *, skip_only_on_ci: bool = False)
     def skipif_platform_linux_because(reason: str, *, skip_only_on_ci: bool = False) -> pytest.MarkDecorator:
         return _make_skipif_platform("linux", reason, skip_only_on_ci=skip_only_on_ci)
 
+    @staticmethod
+    def skipif_platform_bsd_because(reason: str, *, skip_only_on_ci: bool = False) -> pytest.MarkDecorator:
+        return _make_skipif_platform(("freebsd", "openbsd", "netbsd"), reason, skip_only_on_ci=skip_only_on_ci)
+
     skipif_platform_win32 = skipif_platform_win32_because("cannot run on Windows")
     skipif_platform_macOS = skipif_platform_macOS_because("cannot run on MacOS")
     skipif_platform_linux = skipif_platform_linux_because("cannot run on Linux")
+    skipif_platform_bsd = skipif_platform_bsd_because("Cannot run on BSD-related platforms (e.g. FreeBSD)")
 
     ###### RESTRICT TESTS FOR PLATFORMS ######
 

tox.ini

@@ -67,11 +67,16 @@ commands =
     docstrings: pytest --doctest-modules {posargs} {[docs]examples_dir}{/}tutorials{/}ftp_server
     docstrings: pytest --doctest-glob="*.rst" {posargs} {[docs]source_dir}
 
-[testenv:{py311,py312,py313}-{unit,functional}-{standard,cbor,msgpack,trio}]
+[testenv:{py311,py312,py313}-{unit,functional}-{standard,cbor,msgpack,trio}{,-bsd}]
 package = wheel
 wheel_build_env = {[base]wheel_build_env}
+platform =
+    !bsd: ((?!bsd).)*
+    bsd: (free|open|net)bsd.*
 groups =
     test
+    ; Tests with SSL are deactivated on BSD because cryptography takes 4 minutes to compile :)
+    functional-standard-!bsd: test-ssl
     coverage
     cbor: cbor
     msgpack: msgpack
@@ -96,14 +101,15 @@ commands =
     msgpack: pytest -m "feature_msgpack" {posargs} {env:TESTS_ROOTDIR}
     trio: pytest -n "{env:PYTEST_MAX_WORKERS:auto}" -m "feature_trio" {posargs} {env:TESTS_ROOTDIR}
 
-[testenv:{py311,py312,py313}-functional-{asyncio_proactor,uvloop}]
+[testenv:{py311,py312,py313}-functional-{asyncio_proactor,uvloop}{,-bsd}]
 package = wheel
 wheel_build_env = {[base]wheel_build_env}
 platform =
     asyncio_proactor: win32
     uvloop: linux|darwin
 groups =
     test
+    test-ssl
     coverage
     uvloop: uvloop
 setenv =
@@ -125,8 +131,8 @@ commands =
 [testenv:coverage-{unit,functional,full}]
 skip_install = true
 depends =
-    unit: {py311,py312,py313}-unit-{standard,cbor,msgpack,trio}
-    functional: {py311,py312,py313}-functional-{standard,cbor,msgpack,trio,asyncio_proactor,uvloop}
+    unit: {py311,py312,py313}-unit-{standard,cbor,msgpack,trio}{,-bsd}
+    functional: {py311,py312,py313}-functional-{standard,cbor,msgpack,trio,asyncio_proactor,uvloop}{,-bsd}
     full: coverage-{unit,functional}
 parallel_show_output =
     full: True
@@ -181,6 +187,7 @@ platform =
 groups =
     mypy
     test: test
+    test: test-ssl
     full,test,micro_benchmarks: cbor
     full,test,micro_benchmarks: msgpack
     full,test,micro_benchmarks: types-msgpack