Move to generated build.zig files (#10)

Calling `zig build-lib` meant we didn't benefit from any caching, and
also meant the Zig Language Server wouldn't provide code completions for
Pydust.

For now, we completely takeover the build.zig file and ask the user's to
add it to .gitignore. At some point however, users may want to manage
their own build.zig file. When that happens we can add a flag to
generated a build.pydust.zig file and have the user's import the
relevant bits into the actual build.zig.

.gitignore

@@ -161,3 +161,5 @@ zig-out/
 /build/
 /build-*/
 /docgen_tmp/
+
+pytest.build.zig

build.zig

@@ -6,23 +6,31 @@ pub fn build(b: *std.Build) void {
 
     const test_step = b.step("test", "Run library tests");
 
-    const configurePython = ConfigurePythonStep.add(b, .{});
-
     const main_tests = b.addTest(.{
         .root_source_file = .{ .path = "pydust/src/pydust.zig" },
         .main_pkg_path = .{ .path = "pydust/src" },
         .target = target,
         .optimize = optimize,
     });
     main_tests.linkLibC();
-    main_tests.addIncludePath(configurePython.getIncludePath());
-    main_tests.addLibraryPath(configurePython.getLibraryPath());
+    main_tests.addIncludePath(.{ .path = getPythonIncludePath(b.allocator) catch @panic("Missing python") });
+    main_tests.addLibraryPath(.{ .path = getPythonLibraryPath(b.allocator) catch @panic("Missing python") });
     main_tests.linkSystemLibrary("python3.11");
     main_tests.addAnonymousModule("pyconf", .{ .source_file = .{ .path = "./pyconf.dummy.zig" } });
 
     const run_main_tests = b.addRunArtifact(main_tests);
     test_step.dependOn(&run_main_tests.step);
 
+    const example_lib = b.addSharedLibrary(.{
+        .name = "examples",
+        .root_source_file = .{ .path = "example/modules.zig" },
+        .main_pkg_path = .{ .path = "example/" },
+        .target = target,
+        .optimize = optimize,
+    });
+    example_lib.addAnonymousModule("pydust", .{ .source_file = .{ .path = "pydust/src/pydust.zig" } });
+    b.installArtifact(example_lib);
+
     // Option for emitting test binary based on the given root source.
     // This is used for debugging as in .vscode/tasks.json
     const test_debug_root = b.option([]const u8, "test-debug-root", "The root path of a file emitted as a binary for use with the debugger");
@@ -33,62 +41,20 @@ pub fn build(b: *std.Build) void {
     }
 }
 
-pub const ConfigurePythonStep = struct {
-    step: std.build.Step,
-    options: Options,
-
-    // Output paths
-    includePath: std.build.GeneratedFile,
-    libPath: std.build.GeneratedFile,
-
-    const Options = struct {
-        pythonExe: []const u8 = "python3",
-    };
-
-    pub fn add(b: *std.Build, options: Options) *ConfigurePythonStep {
-        const self = b.allocator.create(ConfigurePythonStep) catch @panic("OOM");
-        self.* = .{
-            .step = std.build.Step.init(.{
-                .id = .custom,
-                .name = "configure python",
-                .owner = b,
-                .makeFn = ConfigurePythonStep.make,
-            }),
-            .options = options,
-            .includePath = .{ .step = &self.step },
-            .libPath = .{ .step = &self.step },
-        };
-
-        return self;
-    }
-
-    fn make(step: *std.build.Step, prog: *std.Progress.Node) anyerror!void {
-        prog.setName("Configure Python");
-        prog.activate();
-        const self = @fieldParentPtr(ConfigurePythonStep, "step", step);
-
-        const includeResult = try std.process.Child.exec(.{
-            .allocator = step.owner.allocator,
-            .argv = &.{ self.options.pythonExe, "-c", "import sysconfig; print(sysconfig.get_path('include'), end='')" },
-        });
-        defer step.owner.allocator.free(includeResult.stderr);
-        self.includePath.path = includeResult.stdout;
-
-        const libResult = try std.process.Child.exec(.{
-            .allocator = step.owner.allocator,
-            .argv = &.{ self.options.pythonExe, "-c", "import sysconfig; print(sysconfig.get_config_var('LIBDIR'), end='')" },
-        });
-        defer step.owner.allocator.free(libResult.stderr);
-        self.libPath.path = libResult.stdout;
-
-        prog.end();
-    }
-
-    pub fn getIncludePath(self: *ConfigurePythonStep) std.Build.LazyPath {
-        return .{ .generated = &self.includePath };
-    }
+fn getPythonIncludePath(allocator: std.mem.Allocator) ![]const u8 {
+    const includeResult = try std.process.Child.exec(.{
+        .allocator = allocator,
+        .argv = &.{ "python", "-c", "import sysconfig; print(sysconfig.get_path('include'), end='')" },
+    });
+    defer allocator.free(includeResult.stderr);
+    return includeResult.stdout;
+}
 
-    pub fn getLibraryPath(self: *ConfigurePythonStep) std.Build.LazyPath {
-        return .{ .generated = &self.libPath };
-    }
-};
+fn getPythonLibraryPath(allocator: std.mem.Allocator) ![]const u8 {
+    const includeResult = try std.process.Child.exec(.{
+        .allocator = allocator,
+        .argv = &.{ "python", "-c", "import sysconfig; print(sysconfig.get_config_var('LIBDIR'), end='')" },
+    });
+    defer allocator.free(includeResult.stderr);
+    return includeResult.stdout;
+}

pydust/__main__.py

@@ -1,9 +1,6 @@
 import argparse
-import os
-import shutil
-import subprocess
 
-from pydust import config, zigexe
+from pydust import buildzig
 
 parser = argparse.ArgumentParser()
 sub = parser.add_subparsers(dest="command", required=True)
@@ -22,22 +19,7 @@ def main():
 def debug(args):
     """Given an entrypoint file, compile it for test debugging. Placing it in a well-known location."""
     entrypoint = args.entrypoint
-
-    filename = os.path.basename(entrypoint)
-    name, _ext = os.path.splitext(filename)
-
-    ext_module = config.ExtModule(
-        name=name,
-        root=entrypoint,
-        # Not sure how else we could guess this?
-        limited_api=False,
-    )
-
-    with zigexe.build_argv("test", ext_module, optimize="Debug") as argv:
-        subprocess.run(argv, check=True)
-
-    os.makedirs("zig-out/", exist_ok=True)
-    shutil.move(ext_module.test_bin, "zig-out/debug.bin")
+    buildzig.zig_build(["install", f"-Ddebug-root={entrypoint}"])
 
 
 if __name__ == "__main__":

pydust/build.py

@@ -1,16 +1,6 @@
-import shlex
-import subprocess
-
-from pydust import config, zigexe
+from pydust import buildzig
 
 
 def build():
     """The main entry point from Poetry's build script."""
-    pydust_conf = config.load()
-
-    for ext_module in pydust_conf.ext_modules:
-        # TODO(ngates): figure out if we're running as part of a dev install, or an sdist install?
-        with zigexe.build_argv("build-lib", ext_module, optimize="ReleaseSafe") as argv:
-            retcode = subprocess.call(argv)
-            if retcode != 0:
-                raise ValueError(f"Failed to compile Zig: {' '.join(shlex.quote(arg) for arg in argv)}")
+    buildzig.zig_build(["install"])

pydust/buildzig.py

@@ -0,0 +1,183 @@
+import contextlib
+import os
+import subprocess
+import sys
+import sysconfig
+import textwrap
+from typing import TextIO
+
+import pydust
+from pydust import config
+
+# We ship the PyDust Zig library inside our Python package to make it easier
+# for us to auto-configure user projects.
+PYDUST_ROOT = os.path.join(os.path.dirname(pydust.__file__), "src", "pydust.zig")
+
+PYVER_MINOR = ".".join(str(v) for v in sys.version_info[:2])
+PYVER_HEX = f"{sys.hexversion:#010x}"
+PYINC = sysconfig.get_path("include")
+PYLIB = sysconfig.get_config_var("LIBDIR")
+
+
+def zig_build(argv: list[str], build_zig="build.zig"):
+    generate_build_zig(build_zig)
+    subprocess.run(
+        [sys.executable, "-m", "ziglang", "build", "--build-file", build_zig] + argv,
+        check=True,
+    )
+
+
+def generate_build_zig(build_zig_file):
+    """Generate the build.zig file for the current pyproject.toml.
+
+    Initially we were calling `zig build-lib` directly, and this worked fine except it meant we
+    would need to roll our own caching and figure out how to convince ZLS to pick up our dependencies.
+
+    It's therefore easier, at least for now, to generate a build.zig in the project root and add it
+    to the .gitignore. This means ZLS works as expected, we can leverage zig build caching, and the user
+    can inspect the generated file to assist with debugging.
+    """
+    conf = config.load()
+
+    with open(build_zig_file, "w+") as f:
+        b = Writer(f)
+
+        b.writeln('const std = @import("std");')
+        b.writeln()
+
+        with b.block("pub fn build(b: *std.Build) void"):
+            b.write(
+                """
+                const target = b.standardTargetOptions(.{});
+                const optimize = b.standardOptimizeOption(.{});
+
+                const test_step = b.step("test", "Run library tests");
+                """
+            )
+
+            for ext_module in conf.ext_modules:
+                # TODO(ngates): fix the out filename for non-limited modules
+                assert ext_module.limited_api, "Only limited_api is supported for now"
+
+                # For each module, we generate some options, a library, as well as a test runner.
+                with b.block():
+                    b.write(
+                        f"""
+                        // For each Python ext_module, generate a shared library and test runner.
+                        const pyconf = b.addOptions();
+                        pyconf.addOption([:0]const u8, "module_name", "{ext_module.libname}");
+                        pyconf.addOption(bool, "limited_api", {str(ext_module.limited_api).lower()});
+                        pyconf.addOption([:0]const u8, "hexversion", "{PYVER_HEX}");
+
+                        const lib{ext_module.libname} = b.addSharedLibrary(.{{
+                            .name = "{ext_module.libname}",
+                            .root_source_file = .{{ .path = "{ext_module.root}" }},
+                            .main_pkg_path = .{{ .path = "{conf.root}" }},
+                            .target = target,
+                            .optimize = optimize,
+                        }});
+                        configurePythonInclude(lib{ext_module.libname}, pyconf);
+
+                        // Install the shared library within the source tree
+                        const install{ext_module.libname} = b.addInstallFileWithDir(
+                            lib{ext_module.libname}.getEmittedBin(),
+                            .{{ .custom = ".." }},  // Relative to project root: zig-out/../
+                            "{ext_module.install_path}",
+                        );
+                        b.getInstallStep().dependOn(&install{ext_module.libname}.step);
+
+                        const test{ext_module.libname} = b.addTest(.{{
+                            .root_source_file = .{{ .path = "{ext_module.root}" }},
+                            .main_pkg_path = .{{ .path = "{conf.root}" }},
+                            .target = target,
+                            .optimize = optimize,
+                        }});
+                        configurePythonRuntime(test{ext_module.libname}, pyconf);
+
+                        const run_test{ext_module.libname} = b.addRunArtifact(test{ext_module.libname});
+                        test_step.dependOn(&run_test{ext_module.libname}.step);
+                        """
+                    )
+
+            b.write(
+                f"""
+                // Option for emitting test binary based on the given root source. This can be helpful for debugging.
+                const debugRoot = b.option(
+                    []const u8,
+                    "debug-root",
+                    "The root path of a file emitted as a binary for use with the debugger",
+                );
+                if (debugRoot) |root| {{
+                    const pyconf = b.addOptions();
+                    pyconf.addOption([:0]const u8, "module_name", "debug");
+                    pyconf.addOption(bool, "limited_api", false);
+                    pyconf.addOption([:0]const u8, "hexversion", "{PYVER_HEX}");
+
+                    const testdebug = b.addTest(.{{
+                        .root_source_file = .{{ .path = root }},
+                        .main_pkg_path = .{{ .path = "{conf.root}" }},
+                        .target = target,
+                        .optimize = optimize,
+                    }});
+                    configurePythonRuntime(testdebug, pyconf);
+
+                    const debugBin = b.addInstallBinFile(testdebug.getEmittedBin(), "debug.bin");
+                    b.getInstallStep().dependOn(&debugBin.step);
+                }}
+                """
+            )
+
+        b.write(
+            f"""
+            fn configurePythonInclude(compile: *std.Build.CompileStep, pyconf: *std.Build.Step.Options) void {{
+                compile.addAnonymousModule("pydust", .{{
+                    .source_file = .{{ .path = "{PYDUST_ROOT}" }},
+                    .dependencies = &.{{.{{ .name = "pyconf", .module = pyconf.createModule() }}}},
+                }});
+                compile.addIncludePath(.{{ .path = "{PYINC}" }});
+                compile.linkLibC();
+                compile.linker_allow_shlib_undefined = true;
+            }}
+
+            fn configurePythonRuntime(compile: *std.Build.CompileStep, pyconf: *std.Build.Step.Options) void {{
+                configurePythonInclude(compile, pyconf);
+                compile.linkSystemLibrary("python{PYVER_MINOR}");
+                compile.addLibraryPath(.{{ .path =  "{PYLIB}" }});
+            }}
+            """
+        )
+
+
+class Writer:
+    def __init__(self, fileobj: TextIO) -> None:
+        self.f = fileobj
+        self._indent = 0
+
+    @contextlib.contextmanager
+    def indent(self):
+        self._indent += 4
+        yield
+        self._indent -= 4
+
+    @contextlib.contextmanager
+    def block(self, text: str = ""):
+        self.write(text)
+        self.writeln(" {")
+        with self.indent():
+            yield
+        self.writeln()
+        self.writeln("}")
+        self.writeln()
+
+    def write(self, text: str):
+        if "\n" in text:
+            text = textwrap.dedent(text).strip() + "\n\n"
+        self.f.write(textwrap.indent(text, self._indent * " "))
+
+    def writeln(self, text: str = ""):
+        self.write(text)
+        self.f.write("\n")
+
+
+if __name__ == "__main__":
+    generate_build_zig()

pydust/config.py

@@ -17,8 +17,10 @@ def libname(self):
         return self.name.rsplit(".", maxsplit=1)[-1]
 
     @property
-    def install_prefix(self):
-        return os.path.join(*self.name.split(".")[:-1])
+    def install_path(self):
+        # FIXME(ngates): for non-limited API
+        assert self.limited_api, "Only limited API modules are supported right now"
+        return os.path.join(*self.name.split(".")) + ".abi3.so"
 
     @property
     def test_bin(self):