Setup test example (#5)

* Configures pytest to compile our example modules using the same
infrastructure as pydust would for user projects.
* We pass pyconf into pydust and the user's code so we can switch on
various bits of functionality (and configure limited API)

.github/workflows/ci.yml

@@ -64,4 +64,6 @@ jobs:
       - name: Ruff
         run: poetry run ruff .
       - name: Black
-        run: poetry run black .
+        run: poetry run black .
+      - name: Pytest
+        run: poetry run pytest

.gitignore

@@ -5,6 +5,7 @@ __pycache__/
 
 # C extensions
 *.so
+*.so.o
 
 # Distribution / packaging
 .Python

.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+    "[python]": {
+        "editor.defaultFormatter": "ms-python.black-formatter"
+    },
+}

build.zig

@@ -18,6 +18,8 @@ pub fn build(b: *std.Build) void {
     main_tests.addIncludePath(configurePython.getIncludePath());
     main_tests.addLibraryPath(configurePython.getLibraryPath());
     main_tests.linkSystemLibrary("python3.11");
+    main_tests.addAnonymousModule("pyconf", .{ .source_file = .{ .path = "./pyconf.dummy.zig" } });
+
     const run_main_tests = b.addRunArtifact(main_tests);
     test_step.dependOn(&run_main_tests.step);
 

example/modules.zig

@@ -0,0 +1,44 @@
+const std = @import("std");
+const py = @import("pydust");
+
+const Self = @This();
+
+pub const __doc__ =
+    \\A docstring for the example module.
+    \\
+    \\With lots of lines...
+    \\
+    \\One day we'll parse these from Zig doc comments in the AST :)
+;
+
+/// Internal module state can be declared as struct fields.
+///
+/// Default values must be set inline, or in the __new__ function if
+/// they cannot be defaulted at comptime.
+count: u32 = 0,
+name: py.PyString,
+
+pub fn __new__() !Self {
+    return .{ .name = try py.PyString.fromSlice("Nick") };
+}
+
+pub fn hello() !py.PyString {
+    return try py.PyString.fromSlice("Hello!");
+}
+
+pub fn whoami(self: *const Self) !py.PyString {
+    return self.name;
+}
+
+/// Functions taking a "self" parameter are passed the module state.
+pub fn increment(self: *Self) void {
+    self.count += 1;
+}
+
+pub fn count(self: *const Self) u32 {
+    return self.count;
+}
+
+comptime {
+    py.module(@This());
+}

pyconf.dummy.zig

@@ -0,0 +1,3 @@
+/// This pyconf module is used during our own tests to represent the typically auto-generated pyconf module.
+pub const limited_api = true;
+pub const hexversion = "0x030B0000"; // 3.11

pydust/build.py

@@ -1,3 +1,4 @@
+import shlex
 import subprocess
 
 from pydust import config, zigexe
@@ -11,4 +12,4 @@ def build():
         with zigexe.build_argv("build-lib", ext_module) as argv:
             retcode = subprocess.call(argv)
             if retcode != 0:
-                raise ValueError(f"Failed to compile Zig with args {argv}")
+                raise ValueError(f"Failed to compile Zig: {' '.join(shlex.quote(arg) for arg in argv)}")

pydust/config.py

@@ -10,9 +10,7 @@ class ExtModule(BaseModel):
 
     name: str
     root: str
-
-    # TODO(ngates): maybe we can select this to be the minimum supported version?
-    limited_api: bool = False
+    limited_api: bool = True
 
     @property
     def libname(self):

pydust/src/ffi.zig

@@ -1,6 +1,10 @@
 // Export the Limited Python C API for use within PyDust.
+const pyconf = @import("pyconf");
+
 pub usingnamespace @cImport({
-    @cDefine("Py_LIMITED_API", "0x030A0000"); // 3.10
+    if (pyconf.limited_api) {
+        @cDefine("Py_LIMITED_API", pyconf.hexversion);
+    }
     @cDefine("PY_SSIZE_T_CLEAN", {});
     @cInclude("Python.h");
 });

pydust/src/pydust.zig

@@ -62,7 +62,10 @@ pub fn finalize() void {
 }
 
 /// Register a struct as a Python module definition.
-pub fn module(comptime name: [:0]const u8, comptime definition: type) @TypeOf(definition) {
+pub fn module(comptime definition: type) void {
+    const pyconf = @import("pyconf");
+    const name = pyconf.module_name;
+
     var shortname = name;
     if (std.mem.lastIndexOf(u8, name, ".")) |idx| {
         shortname = name[idx + 1 ..];
@@ -75,7 +78,9 @@ pub fn module(comptime name: [:0]const u8, comptime definition: type) @TypeOf(de
     };
     State.addModule(moddef);
     evaluateDeclarations(definition);
-    return definition;
+
+    const wrapped = modules.define(moddef);
+    @export(wrapped.init, .{ .name = "PyInit_" ++ moddef.name, .linkage = .Strong });
 }
 
 /// Register a struct as a Python class definition.
@@ -181,11 +186,3 @@ fn evaluateDeclarations(comptime definition: type) void {
         _ = @field(definition, decl.name);
     }
 }
-
-/// Export PyInit_<modname> C functions into the output object file.
-pub fn exportInitFunctions() void {
-    inline for (State.modules()) |moddef| {
-        const wrapped = modules.define(moddef);
-        @export(wrapped.init, .{ .name = "PyInit_" ++ moddef.name, .linkage = .Strong });
-    }
-}

pydust/zigexe.py

@@ -11,22 +11,38 @@
 # We ship the PyDust Zig library inside our Python package to make it easier
 # for us to auto-configure user projects.
 PYDUST_ROOT = os.path.join(os.path.dirname(pydust.__file__), "src", "pydust.zig")
+PYVER_MINOR = ".".join(str(v) for v in sys.version_info[:2])
 
 Command = Literal["build-lib"] | Literal["test"]
 
 
 @contextlib.contextmanager
 def build_argv(command: Command, ext_module: config.ExtModule):
     """The main entry point from Poetry's build script."""
-    pydust_conf = config.load()
-
     argv = [sys.executable, "-m", "ziglang", command]
     if command == "build-lib":
         argv += ["-dynamic"]
+        # TODO(ngates): create the correct .so filename based on arch
+        os.makedirs(ext_module.install_prefix, exist_ok=True)
+        argv += [f"-femit-bin={os.path.join(ext_module.install_prefix, ext_module.libname + '.abi3.so')}"]
+
+    if command == "test":
+        # Generate the test binary without running it.
+        # For testing, we need to link libpython too.
+        os.makedirs("zig-out/", exist_ok=True)
+        argv += [
+            f"-femit-bin={ext_module.test_bin}",
+            "--test-no-exec",
+            "-L",
+            sysconfig.get_config_var("LIBDIR"),
+            f"-lpython{PYVER_MINOR}",
+        ]
+
+    argv += ["--name", ext_module.libname, ext_module.root]
+
+    # Configure root package
+    argv += ["--main-pkg-path", config.load().root]
 
-    # Setup pydust as a dependency
-    os.path.dirname(__file__)
-    argv += ["--mod", f"pydust::{PYDUST_ROOT}"]
     # Add Python include directory
     argv += [
         "-I",
@@ -35,40 +51,30 @@ def build_argv(command: Command, ext_module: config.ExtModule):
         "-fallow-shlib-undefined",
     ]
 
-    # Configure root package
-    argv += ["--main-pkg-path", pydust_conf.root]
-
-    # Setup a config package to pass information into the build
-    with tempfile.NamedTemporaryFile(prefix="pyconf_", suffix=".zig", mode="w") as pyconf_file:
-        argv += ["--mod", f"pyconf::{pyconf_file.name}"]
-        pyconf_file.write('pub const foo = "bar";\n')
-        pyconf_file.flush()
+    # Link libC
+    argv += ["-lc"]
 
+    with pyconf(ext_module) as pyconf_file:
+        # Setup a pyconf module
+        argv += ["--mod", f"pyconf::{pyconf_file}"]
+        # Setup pydust as a dependency, and allow it to read from pyconf
+        os.path.dirname(__file__)
+        argv += ["--mod", f"pydust:pyconf:{PYDUST_ROOT}"]
         # Add all our deps
         argv += ["--deps", "pydust,pyconf"]
 
-        # For each module, run a zig build
-        argv += ["--name", ext_module.libname, ext_module.root]
-
-        if command == "test":
-            # Generate the test binary without running it.
-            # For testing, we need to link libpython too.
-            os.makedirs("zig-out/", exist_ok=True)
-            argv += [
-                "-femit-bin=" + os.path.join("zig-out", ext_module.libname + ".test.bin"),
-                "--test-no-exec",
-                "-L",
-                sysconfig.get_config_var("LIBDIR"),
-                "-lpython3.11",  # FIXME,
-            ]
-
-        if command == "build-lib":
-            # TODO(ngates): create the correct .so filename based on arch
-            os.makedirs(ext_module.install_prefix, exist_ok=True)
-            argv += [f"-femit-bin={os.path.join(ext_module.install_prefix, ext_module.libname + '.abi3.so')}"]
-
-        # Calculate the Python hex versions
-        if ext_module.limited_api:
-            argv += ["-DPy_LIMITED_API=0x030B0000"]  # 3.11
-
         yield argv
+
+
+@contextlib.contextmanager
+def pyconf(ext_module: config.ExtModule):
+    """Render a config file to pass information into the build."""
+    with tempfile.NamedTemporaryFile(prefix="pyconf_", suffix=".zig", mode="w") as pyconf_file:
+        pyconf_file.write(f'pub const module_name: [:0]const u8 = "{ext_module.name}";\n')
+        pyconf_file.write(f"pub const limited_api: bool = {str(ext_module.limited_api).lower()};\n")
+
+        hexversion = f"{sys.hexversion:#010x}"
+        pyconf_file.write(f'pub const hexversion: [:0]const u8 = "{hexversion}";\n')
+
+        pyconf_file.flush()
+        yield pyconf_file.name

pyproject.toml

@@ -7,6 +7,7 @@ license = "Apache 2.0"
 readme = "README.md"
 packages = [{include = "pydust"}]
 include = ["src"]
+exclude = ["example"]
 
 [tool.poetry.dependencies]
 python = "^3.11"
@@ -18,9 +19,6 @@ pytest = "^7.4.0"
 ruff = "^0.0.286"
 black = "^23.7.0"
 
-[tool.poetry.plugins."pytest11"]
-pydust = "pydust.pytest_plugin"
-
 [tool.black]
 line-length = 120
 
@@ -32,3 +30,14 @@ target-version = "py310"
 [build-system]
 requires = ["poetry-core"]
 build-backend = "poetry.core.masonry.api"
+
+
+# Out test modules
+
+[tool.pydust]
+root = "example/"
+
+
+[[tool.pydust.ext_module]]
+name = "example.modules"
+root = "example/modules.zig"

test/conftest.py

@@ -0,0 +1,6 @@
+from pydust import build
+
+
+def pytest_collection(session):
+    """We use the same pydust build system for our example modules, but we trigger it from a pytest hook."""
+    build.build()

test/test_modules.py

@@ -0,0 +1,20 @@
+from example import modules
+
+
+def test_module_docstring():
+    assert modules.__doc__.startswith("A docstring for the example module.")
+
+
+def test_modules_function():
+    assert modules.hello() == "Hello!"
+
+
+def test_modules_state():
+    assert modules.whoami() == "Nick"
+
+
+def test_modules_mutable_state():
+    assert modules.count() == 0
+    modules.increment()
+    modules.increment()
+    assert modules.count() == 2