Merge pull request #5370 from donn/pyosys_pybind11

pyosys: rewrite using pybind11

Author: mmicko

.editorconfig

@@ -11,6 +11,10 @@ indent_style = space
 indent_size = 2
 trim_trailing_whitespace = false
 
+[*.rst]
+indent_style = space
+indent_size = 3
+
 [*.yml]
 indent_style = space
 indent_size = 2

.github/workflows/wheels.yml

@@ -55,11 +55,6 @@ jobs:
           submodules: true
           persist-credentials: false
       - uses: actions/setup-python@v5
-      - name: Get Boost Source
-        shell: bash
-        run: |
-          mkdir -p boost
-          curl -L https://github.com/boostorg/boost/releases/download/boost-1.86.0/boost-1.86.0-b2-nodocs.tar.gz | tar --strip-components=1 -xzC boost
       - name: Get FFI
         shell: bash
         run: |
@@ -103,21 +98,16 @@ jobs:
           CIBW_BEFORE_ALL: bash ./.github/workflows/wheels/cibw_before_all.sh
           CIBW_ENVIRONMENT: >
             OPTFLAGS=-O3
-            CXXFLAGS=-I./boost/pfx/include
-            LINKFLAGS=-L./boost/pfx/lib
             PKG_CONFIG_PATH=./ffi/pfx/lib/pkgconfig
-            makeFlags='BOOST_PYTHON_LIB=./boost/pfx/lib/libboost_python*.a'
             PATH="$PWD/bison/src:$PATH"
           CIBW_ENVIRONMENT_MACOS: >
             OPTFLAGS=-O3
-            CXXFLAGS=-I./boost/pfx/include
-            LINKFLAGS=-L./boost/pfx/lib
             PKG_CONFIG_PATH=./ffi/pfx/lib/pkgconfig
             MACOSX_DEPLOYMENT_TARGET=11
-            makeFlags='BOOST_PYTHON_LIB=./boost/pfx/lib/libboost_python*.a CONFIG=clang'
+            makeFlags='CONFIG=clang'
             PATH="$PWD/bison/src:$PATH"
           CIBW_BEFORE_BUILD: bash ./.github/workflows/wheels/cibw_before_build.sh
-          CIBW_TEST_COMMAND: python3 {project}/tests/arch/ecp5/add_sub.py
+          CIBW_TEST_COMMAND: python3 {project}/tests/pyosys/run_tests.py
       - uses: actions/upload-artifact@v4
         with:
           name: python-wheels-${{ matrix.os.runner }}

.github/workflows/wheels/cibw_before_build.sh

@@ -1,8 +1,8 @@
 set -e
 set -x
 
-# Don't use objects from previous compiles on Windows/macOS
-make clean
+# Don't use Python objects from previous compiles
+make clean-py
 
 # DEBUG: show python3 and python3-config outputs
 if [ "$(uname)" != "Linux" ]; then
@@ -11,24 +11,3 @@ if [ "$(uname)" != "Linux" ]; then
 fi
 python3 --version
 python3-config --includes
-
-# Build boost
-cd ./boost
-## Delete the artefacts from previous builds (if any)
-rm -rf ./pfx
-## Bootstrap bjam
-./bootstrap.sh --prefix=./pfx
-## Build Boost against current version of Python, only for
-## static linkage (Boost is statically linked because system boost packages
-## wildly vary in versions, including the libboost_python3 version)
-./b2\
-    -j$(getconf _NPROCESSORS_ONLN 2>/dev/null || sysctl -n hw.ncpu)\
-    --prefix=./pfx\
-    --with-filesystem\
-    --with-system\
-    --with-python\
-    cxxflags="$(python3-config --includes) -std=c++17 -fPIC"\
-    cflags="$(python3-config --includes) -fPIC"\
-    link=static\
-    variant=release\
-    install

CODEOWNERS

@@ -38,7 +38,8 @@ techlibs/gowin/                @pepijndevos
 techlibs/gatemate/             @pu-cc
 
 # pyosys
-misc/*.py                      @btut
+pyosys/*                       @donn
+setup.py                       @donn
 
 backends/firrtl                @ucbjrl @azidar
 

Makefile

@@ -102,7 +102,7 @@ all: top-all
 YOSYS_SRC := $(dir $(firstword $(MAKEFILE_LIST)))
 VPATH := $(YOSYS_SRC)
 
-CXXSTD ?= c++17
+export CXXSTD ?= c++17
 CXXFLAGS := $(CXXFLAGS) -Wall -Wextra -ggdb -I. -I"$(YOSYS_SRC)" -MD -MP -D_YOSYS_ -fPIC -I$(PREFIX)/include
 LIBS := $(LIBS) -lstdc++ -lm
 PLUGIN_LINKFLAGS :=
@@ -133,10 +133,6 @@ LINKFLAGS += -rdynamic
 ifneq ($(shell :; command -v brew),)
 BREW_PREFIX := $(shell brew --prefix)/opt
 $(info $$BREW_PREFIX is [${BREW_PREFIX}])
-ifeq ($(ENABLE_PYOSYS),1)
-CXXFLAGS += -I$(BREW_PREFIX)/boost/include
-LINKFLAGS += -L$(BREW_PREFIX)/boost/lib -L$(BREW_PREFIX)/boost-python3/lib
-endif
 CXXFLAGS += -I$(BREW_PREFIX)/readline/include -I$(BREW_PREFIX)/flex/include
 LINKFLAGS += -L$(BREW_PREFIX)/readline/lib -L$(BREW_PREFIX)/flex/lib
 PKG_CONFIG_PATH := $(BREW_PREFIX)/libffi/lib/pkgconfig:$(PKG_CONFIG_PATH)
@@ -348,31 +344,24 @@ ifeq ($(ENABLE_LIBYOSYS),1)
 TARGETS += libyosys.so
 endif
 
+PY_WRAPPER_FILE = pyosys/wrappers
+
+# running make clean on just those and then recompiling saves a lot of
+# time when running cibuildwheel
+PYTHON_OBJECTS = pyosys/wrappers.o kernel/drivers.o kernel/yosys.o passes/cmds/plugin.o
+
 ifeq ($(ENABLE_PYOSYS),1)
 # python-config --ldflags includes -l and -L, but LINKFLAGS is only -L
 LINKFLAGS += $(filter-out -l%,$(shell $(PYTHON_CONFIG) --ldflags))
 LIBS += $(shell $(PYTHON_CONFIG) --libs)
 EXE_LIBS += $(filter-out $(LIBS),$(shell $(PYTHON_CONFIG_FOR_EXE) --libs))
-CXXFLAGS += $(shell $(PYTHON_CONFIG) --includes) -DWITH_PYTHON
-
-# Detect name of boost_python library. Some distros use boost_python-py<version>, other boost_python<version>, some only use the major version number, some a concatenation of major and minor version numbers
-CHECK_BOOST_PYTHON = (echo "int main(int argc, char ** argv) {return 0;}" | $(CXX) -xc -o /dev/null $(LINKFLAGS) $(EXE_LIBS) $(LIBS) -l$(1) - > /dev/null 2>&1 && echo "-l$(1)")
-BOOST_PYTHON_LIB ?= $(shell \
-	$(call CHECK_BOOST_PYTHON,boost_python-py$(subst .,,$(PYTHON_VERSION))) || \
-	$(call CHECK_BOOST_PYTHON,boost_python-py$(PYTHON_MAJOR_VERSION)) || \
-	$(call CHECK_BOOST_PYTHON,boost_python$(subst .,,$(PYTHON_VERSION))) || \
-	$(call CHECK_BOOST_PYTHON,boost_python$(PYTHON_MAJOR_VERSION)) \
-)
+PYBIND11_INCLUDE ?= $(shell $(PYTHON_EXECUTABLE) -m pybind11 --includes)
+CXXFLAGS += -I$(PYBIND11_INCLUDE) -DYOSYS_ENABLE_PYTHON
+CXXFLAGS += $(shell $(PYTHON_CONFIG) --includes) -DYOSYS_ENABLE_PYTHON
 
-ifeq ($(BOOST_PYTHON_LIB),)
-$(error BOOST_PYTHON_LIB could not be detected. Please define manually)
-endif
-
-LIBS += $(BOOST_PYTHON_LIB) -lboost_system -lboost_filesystem
-PY_WRAPPER_FILE = kernel/python_wrappers
 OBJS += $(PY_WRAPPER_FILE).o
-PY_GEN_SCRIPT= py_wrap_generator
-PY_WRAP_INCLUDES := $(shell $(PYTHON_EXECUTABLE) -c "from misc import $(PY_GEN_SCRIPT); $(PY_GEN_SCRIPT).print_includes()")
+PY_GEN_SCRIPT = pyosys/generator.py
+PY_WRAP_INCLUDES := $(shell $(PYTHON_EXECUTABLE) $(PY_GEN_SCRIPT) --print-includes)
 endif # ENABLE_PYOSYS
 
 ifeq ($(ENABLE_READLINE),1)
@@ -786,9 +775,9 @@ endif
 	$(P) cat $< | grep -E -v "#[ ]*(include|error)" | $(CXX) $(CXXFLAGS) -x c++ -o $@ -E -P -
 
 ifeq ($(ENABLE_PYOSYS),1)
-$(PY_WRAPPER_FILE).cc: misc/$(PY_GEN_SCRIPT).py $(PY_WRAP_INCLUDES)
+$(PY_WRAPPER_FILE).cc: $(PY_GEN_SCRIPT) pyosys/wrappers_tpl.cc $(PY_WRAP_INCLUDES) pyosys/hashlib.h
 	$(Q) mkdir -p $(dir $@)
-	$(P) $(PYTHON_EXECUTABLE) -c "from misc import $(PY_GEN_SCRIPT); $(PY_GEN_SCRIPT).gen_wrappers(\"$(PY_WRAPPER_FILE).cc\")"
+	$(P) $(PYTHON_EXECUTABLE) $(PY_GEN_SCRIPT) $(PY_WRAPPER_FILE).cc
 endif
 
 %.o: %.cpp
@@ -1035,12 +1024,12 @@ ifeq ($(ENABLE_LIBYOSYS),1)
 	if [ -n "$(STRIP)" ]; then $(INSTALL_SUDO) $(STRIP) -S $(DESTDIR)$(LIBDIR)/libyosys.so; fi
 ifeq ($(ENABLE_PYOSYS),1)
 	$(INSTALL_SUDO) mkdir -p $(DESTDIR)$(PYTHON_DESTDIR)/$(subst -,_,$(PROGRAM_PREFIX))pyosys
+	$(INSTALL_SUDO) cp pyosys/__init__.py $(DESTDIR)$(PYTHON_DESTDIR)/$(subst -,_,$(PROGRAM_PREFIX))pyosys/__init__.py
 	$(INSTALL_SUDO) cp libyosys.so $(DESTDIR)$(PYTHON_DESTDIR)/$(subst -,_,$(PROGRAM_PREFIX))pyosys/libyosys.so
 	$(INSTALL_SUDO) cp -r share $(DESTDIR)$(PYTHON_DESTDIR)/$(subst -,_,$(PROGRAM_PREFIX))pyosys
 ifeq ($(ENABLE_ABC),1)
 	$(INSTALL_SUDO) cp yosys-abc $(DESTDIR)$(PYTHON_DESTDIR)/$(subst -,_,$(PROGRAM_PREFIX))pyosys/yosys-abc
 endif
-	$(INSTALL_SUDO) cp misc/__init__.py $(DESTDIR)$(PYTHON_DESTDIR)/$(subst -,_,$(PROGRAM_PREFIX))pyosys/
 endif
 endif
 ifeq ($(ENABLE_PLUGINS),1)
@@ -1130,12 +1119,10 @@ DOC_TARGET ?= html
 docs: docs/prep
 	$(Q) $(MAKE) -C docs $(DOC_TARGET)
 
-clean:
+clean: clean-py
 	rm -rf share
-	rm -rf kernel/*.pyh
-	rm -f $(OBJS) $(GENFILES) $(TARGETS) $(EXTRA_TARGETS) $(EXTRA_OBJS) $(PY_WRAP_INCLUDES) $(PY_WRAPPER_FILE).cc
+	rm -f $(OBJS) $(GENFILES) $(TARGETS) $(EXTRA_TARGETS) $(EXTRA_OBJS) $(PY_WRAP_INCLUDES)
 	rm -f kernel/version_*.o kernel/version_*.cc
-	rm -f kernel/python_wrappers.o
 	rm -f libs/*/*.d frontends/*/*.d passes/*/*.d backends/*/*.d kernel/*.d techlibs/*/*.d
 	rm -rf tests/asicworld/*.out tests/asicworld/*.log
 	rm -rf tests/hana/*.out tests/hana/*.log
@@ -1149,8 +1136,14 @@ clean:
 	rm -f $(addsuffix /run-test.mk,$(MK_TEST_DIRS))
 	-$(MAKE) -C docs clean
 	rm -rf docs/util/__pycache__
+	rm -f libyosys.so
+
+clean-py:
+	rm -f $(PY_WRAPPER_FILE).inc.cc $(PY_WRAPPER_FILE).cc
+	rm -f $(PYTHON_OBJECTS)
 	rm -f *.whl
 	rm -f libyosys.so
+	rm -rf kernel/*.pyh
 
 clean-abc:
 	$(MAKE) -C abc DEP= clean

docs/.gitignore

@@ -6,3 +6,4 @@
 /source/_images/**/*.svg
 /source/_images/**/*.dot
 /source/_images/code_examples
+/venv

docs/source/code_examples/pyosys/pass.py

@@ -0,0 +1,37 @@
+from pyosys import libyosys as ys
+
+class AllEnablePass(ys.Pass):
+	def __init__(self):
+		super().__init__(
+			"all_enable",
+			"makes all _DFF_P_ registers require an enable signal"
+		)
+
+	def execute(self, args, design):
+		ys.log_header(design, "Adding enable signals\n")
+		ys.log_push()
+		top_module = design.top_module()
+
+		if "\\enable" not in top_module.wires_:
+			enable_line = top_module.addWire("\\enable")
+			enable_line.port_input = True
+			top_module.fixup_ports()
+
+		for cell in top_module.cells_.values():
+			if cell.type != "$_DFF_P_":
+				continue
+			cell.type = "$_DFFE_PP_"
+			cell.setPort("\\E", ys.SigSpec(enable_line))
+		ys.log_pop()
+
+p = AllEnablePass() # register the pass
+
+# using the pass
+
+design = ys.Design()
+ys.run_pass("read_verilog tests/simple/fiedler-cooley.v", design)
+ys.run_pass("hierarchy -check -auto-top", design)
+ys.run_pass("synth", design)
+ys.run_pass("all_enable", design)
+ys.run_pass("write_verilog out.v", design)
+ys.run_pass("synth_ice40 -json out.json", design)

docs/source/code_examples/pyosys/simple_database.py

@@ -0,0 +1,51 @@
+from pyosys import libyosys as ys
+
+# loading design
+design = ys.Design()
+
+ys.run_pass("read_verilog tests/simple/fiedler-cooley.v", design)
+ys.run_pass("hierarchy -check -auto-top", design)
+
+# top module inspection
+top_module = design.top_module()
+
+for id, wire in top_module.wires_.items():
+	if not wire.port_input and not wire.port_output:
+		continue
+	description = "input" if wire.port_input else "output"
+	description += " " + wire.name.str()
+	if wire.width != 1:
+		frm = wire.start_offset
+		to = wire.start_offset + wire.width
+		if wire.upto:
+			to, frm = frm, to
+		description += f" [{to}:{frm}]"
+	print(description)
+
+# synth
+
+ys.run_pass("synth", design)
+
+# adding the enable line
+
+enable_line = top_module.addWire("\\enable")
+enable_line.port_input = True
+top_module.fixup_ports()
+
+# hooking the enable line to the internal dff cells
+
+for cell in top_module.cells_.values():
+	if cell.type != "$_DFF_P_":
+		continue
+	cell.type = "$_DFFE_PP_"
+	cell.setPort("\\E", ys.SigSpec(enable_line))
+
+# run check
+
+top_module.check()
+ys.run_pass("stat", design)
+
+# write outputs
+
+ys.run_pass("write_verilog out.v", design)
+ys.run_pass("synth_ice40 -json out.json", design)

docs/source/using_yosys/index.rst

@@ -17,3 +17,4 @@ ways Yosys can interact with designs for a deeper investigation.
    more_scripting/index
    bugpoint
    verilog
+   pyosys