simplified new build system using build/plugins.mk

eliminate repetition and typos by putting all build time flags into a single place, supports COMPILER and CCACHE for now. update CI worlflows to use new build system standalone build of zencode-exec target for bindings for python, rust and golang only the zencode-exec binary is needed
dyne · Sep 24, 2024 · 6011cad · 6011cad
1 parent 776d86b
commit 6011cad
Show file tree

Hide file tree

Showing 11 changed files with 113 additions and 115 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -173,11 +173,12 @@ jobs:
     if: "!contains(github.event.pull_request.labels.*.name, 'SKIP_GO')"
     strategy:
       matrix:
-        go: ['1.21', '1.22']
+        go: ['1.23.1', '1.22.7']
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
+      - uses: hendrikmuhs/[email protected]
       - uses: actions/setup-go@v5
         with:
           go-version: ${{ matrix.go }}
@@ -187,10 +188,10 @@ jobs:
           python-version: '3.10'
       - run: |
           pip3 install meson ninja
-          sudo apt-get install --force-yes zsh libreadline-dev
+          sudo apt-get install -qy zsh libreadline-dev
       - run: |
-          make linux
-      - run: cp src/zencode-exec /usr/local/bin/
+          make -f build/linux.mk deps zencode-exec CCACHE=1
+          cp zencode-exec /usr/local/bin/
       - name: 🧪 test bindings go-${{ matrix.go }}
         working-directory: bindings/golang/zenroom
         run: make test
@@ -209,14 +210,16 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
+      - uses: hendrikmuhs/[email protected]
       - uses: dtolnay/rust-toolchain@stable
         with:
           toolchain: ${{ matrix.rust }}
       - run: |
           pip3 install meson ninja
-          sudo apt-get install --force-yes zsh libreadline-dev
-      - run: make linux
-      - run: cp src/zencode-exec /usr/local/bin/
+          sudo apt-get install -qy zsh libreadline-dev
+      - run: |
+          make -f build/linux.mk CCACHE=1
+          cp zencode-exec /usr/local/bin/
       - name: 🧪 test bindings rust-${{ matrix.rust }}
         working-directory: bindings/rust
         run: cargo test
@@ -266,16 +269,25 @@ jobs:
       - uses: mymindstorm/setup-emsdk@v14
         with:
           version: ${{ env.EM_VERSION }}
-      - run: sudo apt-get update; sudo apt-get install zsh meson ninja-build
-      - run: yarn --cwd bindings/javascript
-      - run: yarn --cwd bindings/javascript build
+      - name: Install dependencies
+        run: |
+          sudo apt-get -qy update
+          sudo apt-get install -qy zsh meson ninja-build
+      - name: Install yarn dependencies
+        run: |
+          yarn --cwd bindings/javascript
+      - name: Build Zenroom Wasm
+        run: |
+          yarn --cwd bindings/javascript build
         env:
           EMSCRIPTEN: ${{ env.EMSDK }}/upstream/emscripten
       - name: 🧪 test bindings node-${{ matrix.node-version }}
-        run: yarn --cwd bindings/javascript test
+        run: |
+          yarn --cwd bindings/javascript test
       - name: 🔐 check crypto with node-${{ matrix.node-version }}
         working-directory: ./
-        run: make check-js
+        run: |
+          make check-js
 
   python-check:
     name: 🐍 Python checks
@@ -297,16 +309,27 @@ jobs:
       - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-      - run: |
+      - name: Install dependencies on Linux
+        run: |
           sudo apt update
-          sudo apt install -y zsh cmake libreadline-dev
+          sudo apt install -yq zsh cmake libreadline-dev
         if: runner.os == 'Linux'
-      - run: brew install zsh
+      - name: Install dependencies on Apple/OSX
+        run: |
+          brew install zsh
         if: runner.os == 'macOS'
-      - run: |
+      - name: Build zencode-exec on Linux
+        run: |
+          cd ../..
+          make -f build/linux.mk deps zencode-exec
+          cp zencode-exec /usr/local/bin/
+        if: runner.os == 'Linux'
+      - name: Build zencode-exec on Apple/OSX
+        run: |
           cd ../..
-          make linux
-          cp src/zencode-exec /usr/local/bin/
+          make -f build/apple-osx.mk deps zencode-exec.command
+          cp zencode-exec.command /usr/local/bin/zencode-exec
+        if: runner.os == 'macOS'
       - run: |
           ./prepare.sh
           pipx install meson
@@ -362,9 +385,16 @@ jobs:
       - uses: mymindstorm/setup-emsdk@v14
         with:
           version: ${{ env.EM_VERSION }}
-      - run: sudo apt-get update; sudo apt-get install zsh meson ninja-build
-      - run: yarn --cwd bindings/javascript
-      - run: yarn --cwd bindings/javascript build
+      - name: Install dependencies
+        run: |
+          sudo apt-get -qy update
+          sudo apt-get install -qy zsh meson ninja-build
+      - name: Install yarn dependencies
+        run: |
+          yarn --cwd bindings/javascript
+      - name: Build Zenroom Wasm
+        run: |
+          yarn --cwd bindings/javascript build
         env:
           EMSCRIPTEN: ${{ env.EMSDK }}/upstream/emscripten
       - uses: actions/upload-artifact@v4
@@ -395,23 +425,22 @@ jobs:
             build/release-intro.md
       - name: Install build deps
         run: |
-          sudo apt install zsh tree jq cmake make gcc g++ musl-dev meson clang gcc-mingw-w64 g++-mingw-w64 libreadline-dev ccache
-      - name: Build mimalloc
-        run: make mimalloc
+          sudo apt install -qy zsh tree jq cmake make gcc g++ musl-dev meson clang gcc-mingw-w64 g++-mingw-w64 libreadline-dev ccache
       - name: Build x86_64 with musl-system
-        run: make musl-system
+        run: |
+          make -f build/musl-linux.mk CCACHE=1
       - name: Upload artifact linux-amd64
         uses: actions/upload-artifact@v4
         with:
           name: release-bin-linux-amd64
           path: |
-            src/zenroom
-            src/zencode-exec
+            zenroom
+            zencode-exec
       - name: Clean for next build
         run: make clean
       - name: Build x86_64 shlib
         run: |
-          make linux-meson-clang-release
+          make -f build/meson.mk
       - name: Upload artifact linux-lib-amd64
         uses: actions/upload-artifact@v4
         with:

diff --git a/build/apple-ios.mk b/build/apple-ios.mk
@@ -1,13 +1,8 @@
 ## Initialize build defaults
 include build/init.mk
 
-ifdef CCACHE
-	milagro_cmake_flags += -DCMAKE_C_COMPILER_LAUNCHER=ccache
-	pfxcc += ccache
-endif
-
 OS := iphoneos
-cc := $(shell xcrun --sdk iphoneos -f gcc 2>/dev/null)
+COMPILER := $(shell xcrun --sdk iphoneos -f gcc 2>/dev/null)
 ar := $(shell xcrun --sdk iphoneos -f ar 2>/dev/null)
 ld := $(shell xcrun --sdk iphoneos -f ld 2>/dev/null)
 ranlib := $(shell xcrun --sdk iphoneos -f ranlib 2>/dev/null)
@@ -17,11 +12,9 @@ cflags += -DLIBCMALLOC
 lua_cflags += -DLUA_USE_IOS
 ldflags := -lm
 platform := ios
-quantum_proof_cc := ${pfxcc} ${cc}
-ed25519_cc := ${pfxcc} ${cc}
-libcc_cc := ${pfxcc} ${cc}
-luacc_cc := ${pfxcc} ${cc}
-zenroom_cc := ${pfxcc} ${cc}
+
+# activate CCACHE etc.
+include build/plugins.mk
 
 ios-armv7: cflags += -arch armv7 -isysroot $(shell xcrun --sdk iphoneos --show-sdk-path 2>/dev/null)
 ios-armv7: ${BUILD_DEPS} ${ZEN_SOURCES}
@@ -47,10 +40,4 @@ ios-sim: ${BUILD_DEPS} ${ZEN_SOURCES}
 	TARGET=x86_64 libtool -static -o zenroom-ios-x86_64.a \
 		${ZEN_SOURCES} ${ldadd}
 
-# ios-fat:
-# 	lipo -create \
-# 	zenroom-ios-x86_64.a \
-# 	zenroom-ios-arm64.a \
-# 	-output zenroom-ios.a
-
 include build/deps.mk
diff --git a/build/apple-osx.mk b/build/apple-osx.mk
@@ -1,21 +1,18 @@
 ## Initialize build defaults
 include build/init.mk
 
-ifdef CCACHE
-	milagro_cmake_flags += -DCMAKE_C_COMPILER_LAUNCHER=ccache
-	quantum_proof_cc := ccache ${cc}
-	ed25519_cc := ccache ${cc}
-	libcc_cc := ccache ${cc}
-	lua_cc := ccache ${cc}
-	zenroom_cc := ccache ${cc}
-endif
 ifdef DEBUG
 	cflags += ${cflags_debug}
 else
 	cflags += ${cflags_protection}
 endif
 
-all: ${BUILD_DEPS} zenroom.command zencode-exec.command
+# activate CCACHE etc.
+include build/plugins.mk
+
+all: deps zenroom.command zencode-exec.command
+
+deps: ${BUILD_DEPS}
 
 cli_sources := src/cli-zenroom.o src/repl.o
 zenroom.command: ${ZEN_SOURCES} ${cli_sources}

diff --git a/build/deps.mk b/build/deps.mk
@@ -5,10 +5,9 @@
 	-c $< -o $@ \
 	-DVERSION=\"${VERSION}\" \
 	-DCURRENT_YEAR=\"${CURRENT_YEAR}\" \
-	-DMAKETARGET=\"${MAKECMDGOALS}\" \
 	-DCOMMIT=\"${COMMIT}\" \
 	-DBRANCH=\"${BRANCH}\" \
-	-DCFLAGS="${CFLAGS}"
+	-DCFLAGS="${cflags}"
 
 embed-lua: lua_embed_opts := $(if ${COMPILE_LUA}, compile)
 embed-lua:
@@ -95,6 +94,6 @@ ed25519-donna:
 tinycc-ccache: tinycc
 tinycc:
 	$(info -- Building tinycc embedded C compiler)
-	cd ${pwd}/lib/tinycc && CC="${libcc_cc}" AR=${ar} CFLAGS="${cflags}"	\
+	cd ${pwd}/lib/tinycc && CC="${libtcc_cc}" AR=${ar} CFLAGS="${cflags}"	\
 	LDFLAGS="${ldflags}" ./configure --config-musl && \
 	$(MAKE) libtcc.a libtcc1.a
diff --git a/build/init.mk b/build/init.mk
@@ -1,4 +1,4 @@
-BRANCH := $(shell git symbolic-ref HEAD | sed -e 's,.*/\(.*\),\1,')
+BRANCH := $(shell git symbolic-ref HEAD | cut -d/ -f3-)
 COMMIT := $(shell git rev-parse --short HEAD)
 VERSION := $(shell git describe --tags | cut -d- -f1)
 CURRENT_YEAR := $(shell date +%Y)
@@ -102,4 +102,5 @@ mimalloc_cflags += -fvisibility=hidden -Wstrict-prototypes
 mimalloc_cflags += -ftls-model=initial-exec -fno-builtin-malloc
 mimalloc_cflags += -DMI_USE_RTLGENRANDOM
 
+
 # }}}
diff --git a/build/linux.mk b/build/linux.mk
@@ -3,13 +3,6 @@ include build/init.mk
 
 COMPILER ?= gcc
 
-cc := ${COMPILER}
-quantum_proof_cc := ${cc}
-ed25519_cc := ${cc}
-libcc_cc := ${cc}
-lua_cc := ${cc}
-zenroom_cc := ${cc}
-
 ## Additional dependencies
 BUILD_DEPS += tinycc mimalloc
 ldadd += -lm
@@ -19,22 +12,19 @@ ldadd += ${pwd}/lib/mimalloc/build/libmimalloc-static.a
 cflags += -fPIC -D'ARCH="LINUX"' -DARCH_LINUX
 system := Linux
 
-ifdef CCACHE
-	milagro_cmake_flags += -DCMAKE_C_COMPILER_LAUNCHER=ccache
-	quantum_proof_cc := ccache ${cc}
-	ed25519_cc := ccache ${cc}
-	libcc_cc := ccache ${cc}
-	lua_cc := ccache ${cc}
-	zenroom_cc := ccache ${cc}
-endif
 # default is DEBUG
 ifdef RELEASE
 	cflags += -O3 ${cflags_protection}
 else
 	cflags += ${cflags_debug}
 endif
 
-all: ${BUILD_DEPS} zenroom zencode-exec zencc
+# activate CCACHE etc.
+include build/plugins.mk
+
+all: deps zenroom zencode-exec zencc
+
+deps: ${BUILD_DEPS}
 
 cli_sources := src/cli-zenroom.o src/repl.o
 zenroom: ${ZEN_SOURCES} ${cli_sources}

diff --git a/build/meson.mk b/build/meson.mk
@@ -3,28 +3,18 @@ include build/init.mk
 
 BUILD_DEPS += mimalloc
 BUILD_DEPS += tinycc
+
+COMPILER ?= gcc
+
 ## Specific compiler settings for all built dependencies
 ifdef RELEASE
 	cflags +=  -O3 ${cflags_protection} -fPIC
 else
 	cflags += ${cflags_debug} -fPIC
 endif
-ifdef CLANG
-	cc := clang
-	zenroom_cc := ${cc}
-	quantum_proof_cc := ${cc}
-	ed25519_cc := ${cc}
-	lua_cc := ${cc}
-endif
-ifdef CCACHE
-	milagro_cmake_flags += -DCMAKE_C_COMPILER_LAUNCHER=ccache
-	mimalloc_cmake_flags += -DCMAKE_C_COMPILER_LAUNCHER=ccache
-	mimalloc_cmake_flags += -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
-	zenroom_cc := ccache ${cc}
-	quantum_proof_cc := ccache ${cc}
-	ed25519_cc := ccache ${cc}
-	lua_cc := ccache ${cc}
-endif
+
+# activate CCACHE etc.
+include build/plugins.mk
 
 # MAIN TARGETS
 all: ${BUILD_DEPS} prepare

diff --git a/build/musl-linux.mk b/build/musl-linux.mk
@@ -1,12 +1,7 @@
 ## Initialize build defaults
 include build/init.mk
 
-cc := musl-gcc
-quantum_proof_cc := ${cc}
-ed25519_cc := ${cc}
-libcc_cc := ${cc}
-lua_cc := ${cc}
-zenroom_cc := ${cc}
+COMPILER := musl-gcc
 
 ## Additional dependencies
 BUILD_DEPS += tinycc
@@ -16,20 +11,15 @@ cflags += -static -std=gnu99 -fPIC -D'ARCH="MUSL"' -D__MUSL__ -DARCH_MUSL
 ldflags += -static
 system := Linux
 
-ifdef CCACHE
-	milagro_cmake_flags += -DCMAKE_C_COMPILER_LAUNCHER=ccache
-	quantum_proof_cc := ccache ${cc}
-	ed25519_cc := ccache ${cc}
-	libcc_cc := ccache ${cc}
-	lua_cc := ccache ${cc}
-	zenroom_cc := ccache ${cc}
-endif
 ifdef DEBUG
 	cflags += ${cflags_debug}
 else
 	cflags += -O3 ${cflags_protection}
 endif
 
+# activate CCACHE etc.
+include build/plugins.mk
+
 all: ${BUILD_DEPS} zenroom zencode-exec zencc
 
 cli_sources := src/cli-zenroom.o src/repl.o