From 7e773101b6b032f7ad6b28cd3d551ffd5ad35d7c Mon Sep 17 00:00:00 2001 From: Jaromil Date: Tue, 24 Sep 2024 10:07:24 +0200 Subject: [PATCH] simplified new build system using build/plugins.mk eliminate repetition and typos by putting all build time flags into a single place, supports COMPILER and CCACHE for now. update CI worlflows to use new build system standalone build of zencode-exec target for bindings for python, rust and golang only the zencode-exec binary is needed --- .github/workflows/main.yml | 85 +++++++++++++++++++++++++------------- build/apple-ios.mk | 21 ++-------- build/apple-osx.mk | 15 +++---- build/deps.mk | 5 +-- build/init.mk | 3 +- build/linux.mk | 22 +++------- build/meson.mk | 22 +++------- build/musl-linux.mk | 18 ++------ build/plugins.mk | 20 +++++++++ build/win-exe.mk | 14 ++----- lib/pqclean/Makefile | 3 ++ 11 files changed, 113 insertions(+), 115 deletions(-) create mode 100644 build/plugins.mk diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c5ad4ef58..1996e2f19 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -173,11 +173,12 @@ jobs: if: "!contains(github.event.pull_request.labels.*.name, 'SKIP_GO')" strategy: matrix: - go: ['1.21', '1.22'] + go: ['1.23.1', '1.22.7'] steps: - uses: actions/checkout@v4 with: fetch-depth: 0 + - uses: hendrikmuhs/ccache-action@v1.2 - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} @@ -187,10 +188,10 @@ jobs: python-version: '3.10' - run: | pip3 install meson ninja - sudo apt-get install --force-yes zsh libreadline-dev + sudo apt-get install -qy zsh libreadline-dev - run: | - make linux - - run: cp src/zencode-exec /usr/local/bin/ + make -f build/linux.mk deps zencode-exec CCACHE=1 + cp zencode-exec /usr/local/bin/ - name: ๐Ÿงช test bindings go-${{ matrix.go }} working-directory: bindings/golang/zenroom run: make test @@ -209,14 +210,16 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 + - uses: hendrikmuhs/ccache-action@v1.2 - uses: dtolnay/rust-toolchain@stable with: toolchain: ${{ matrix.rust }} - run: | pip3 install meson ninja - sudo apt-get install --force-yes zsh libreadline-dev - - run: make linux - - run: cp src/zencode-exec /usr/local/bin/ + sudo apt-get install -qy zsh libreadline-dev + - run: | + make -f build/linux.mk CCACHE=1 + cp zencode-exec /usr/local/bin/ - name: ๐Ÿงช test bindings rust-${{ matrix.rust }} working-directory: bindings/rust run: cargo test @@ -266,16 +269,25 @@ jobs: - uses: mymindstorm/setup-emsdk@v14 with: version: ${{ env.EM_VERSION }} - - run: sudo apt-get update; sudo apt-get install zsh meson ninja-build - - run: yarn --cwd bindings/javascript - - run: yarn --cwd bindings/javascript build + - name: Install dependencies + run: | + sudo apt-get -qy update + sudo apt-get install -qy zsh meson ninja-build + - name: Install yarn dependencies + run: | + yarn --cwd bindings/javascript + - name: Build Zenroom Wasm + run: | + yarn --cwd bindings/javascript build env: EMSCRIPTEN: ${{ env.EMSDK }}/upstream/emscripten - name: ๐Ÿงช test bindings node-${{ matrix.node-version }} - run: yarn --cwd bindings/javascript test + run: | + yarn --cwd bindings/javascript test - name: ๐Ÿ” check crypto with node-${{ matrix.node-version }} working-directory: ./ - run: make check-js + run: | + make check-js python-check: name: ๐Ÿ Python checks @@ -297,16 +309,27 @@ jobs: - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - run: | + - name: Install dependencies on Linux + run: | sudo apt update - sudo apt install -y zsh cmake libreadline-dev + sudo apt install -yq zsh cmake libreadline-dev if: runner.os == 'Linux' - - run: brew install zsh + - name: Install dependencies on Apple/OSX + run: | + brew install zsh if: runner.os == 'macOS' - - run: | + - name: Build zencode-exec on Linux + run: | + cd ../.. + make -f build/linux.mk deps zencode-exec + cp zencode-exec /usr/local/bin/ + if: runner.os == 'Linux' + - name: Build zencode-exec on Apple/OSX + run: | cd ../.. - make linux - cp src/zencode-exec /usr/local/bin/ + make -f build/apple-osx.mk deps zencode-exec.command + cp zencode-exec.command /usr/local/bin/zencode-exec + if: runner.os == 'macOS' - run: | ./prepare.sh pipx install meson @@ -362,9 +385,16 @@ jobs: - uses: mymindstorm/setup-emsdk@v14 with: version: ${{ env.EM_VERSION }} - - run: sudo apt-get update; sudo apt-get install zsh meson ninja-build - - run: yarn --cwd bindings/javascript - - run: yarn --cwd bindings/javascript build + - name: Install dependencies + run: | + sudo apt-get -qy update + sudo apt-get install -qy zsh meson ninja-build + - name: Install yarn dependencies + run: | + yarn --cwd bindings/javascript + - name: Build Zenroom Wasm + run: | + yarn --cwd bindings/javascript build env: EMSCRIPTEN: ${{ env.EMSDK }}/upstream/emscripten - uses: actions/upload-artifact@v4 @@ -395,23 +425,22 @@ jobs: build/release-intro.md - name: Install build deps run: | - sudo apt install zsh tree jq cmake make gcc g++ musl-dev meson clang gcc-mingw-w64 g++-mingw-w64 libreadline-dev ccache - - name: Build mimalloc - run: make mimalloc + sudo apt install -qy zsh tree jq cmake make gcc g++ musl-dev meson clang gcc-mingw-w64 g++-mingw-w64 libreadline-dev ccache - name: Build x86_64 with musl-system - run: make musl-system + run: | + make -f build/musl-linux.mk CCACHE=1 - name: Upload artifact linux-amd64 uses: actions/upload-artifact@v4 with: name: release-bin-linux-amd64 path: | - src/zenroom - src/zencode-exec + zenroom + zencode-exec - name: Clean for next build run: make clean - name: Build x86_64 shlib run: | - make linux-meson-clang-release + make -f build/meson.mk - name: Upload artifact linux-lib-amd64 uses: actions/upload-artifact@v4 with: diff --git a/build/apple-ios.mk b/build/apple-ios.mk index 68a310440..ffbceb9a4 100644 --- a/build/apple-ios.mk +++ b/build/apple-ios.mk @@ -1,13 +1,8 @@ ## Initialize build defaults include build/init.mk -ifdef CCACHE - milagro_cmake_flags += -DCMAKE_C_COMPILER_LAUNCHER=ccache - pfxcc += ccache -endif - OS := iphoneos -cc := $(shell xcrun --sdk iphoneos -f gcc 2>/dev/null) +COMPILER := $(shell xcrun --sdk iphoneos -f gcc 2>/dev/null) ar := $(shell xcrun --sdk iphoneos -f ar 2>/dev/null) ld := $(shell xcrun --sdk iphoneos -f ld 2>/dev/null) ranlib := $(shell xcrun --sdk iphoneos -f ranlib 2>/dev/null) @@ -17,11 +12,9 @@ cflags += -DLIBCMALLOC lua_cflags += -DLUA_USE_IOS ldflags := -lm platform := ios -quantum_proof_cc := ${pfxcc} ${cc} -ed25519_cc := ${pfxcc} ${cc} -libcc_cc := ${pfxcc} ${cc} -luacc_cc := ${pfxcc} ${cc} -zenroom_cc := ${pfxcc} ${cc} + +# activate CCACHE etc. +include build/plugins.mk ios-armv7: cflags += -arch armv7 -isysroot $(shell xcrun --sdk iphoneos --show-sdk-path 2>/dev/null) ios-armv7: ${BUILD_DEPS} ${ZEN_SOURCES} @@ -47,10 +40,4 @@ ios-sim: ${BUILD_DEPS} ${ZEN_SOURCES} TARGET=x86_64 libtool -static -o zenroom-ios-x86_64.a \ ${ZEN_SOURCES} ${ldadd} -# ios-fat: -# lipo -create \ -# zenroom-ios-x86_64.a \ -# zenroom-ios-arm64.a \ -# -output zenroom-ios.a - include build/deps.mk diff --git a/build/apple-osx.mk b/build/apple-osx.mk index bcae4d549..37825d8d8 100644 --- a/build/apple-osx.mk +++ b/build/apple-osx.mk @@ -1,21 +1,18 @@ ## Initialize build defaults include build/init.mk -ifdef CCACHE - milagro_cmake_flags += -DCMAKE_C_COMPILER_LAUNCHER=ccache - quantum_proof_cc := ccache ${cc} - ed25519_cc := ccache ${cc} - libcc_cc := ccache ${cc} - lua_cc := ccache ${cc} - zenroom_cc := ccache ${cc} -endif ifdef DEBUG cflags += ${cflags_debug} else cflags += ${cflags_protection} endif -all: ${BUILD_DEPS} zenroom.command zencode-exec.command +# activate CCACHE etc. +include build/plugins.mk + +all: deps zenroom.command zencode-exec.command + +deps: ${BUILD_DEPS} cli_sources := src/cli-zenroom.o src/repl.o zenroom.command: ${ZEN_SOURCES} ${cli_sources} diff --git a/build/deps.mk b/build/deps.mk index c7e9af086..231d546e6 100644 --- a/build/deps.mk +++ b/build/deps.mk @@ -5,10 +5,9 @@ -c $< -o $@ \ -DVERSION=\"${VERSION}\" \ -DCURRENT_YEAR=\"${CURRENT_YEAR}\" \ - -DMAKETARGET=\"${MAKECMDGOALS}\" \ -DCOMMIT=\"${COMMIT}\" \ -DBRANCH=\"${BRANCH}\" \ - -DCFLAGS="${CFLAGS}" + -DCFLAGS="${cflags}" embed-lua: lua_embed_opts := $(if ${COMPILE_LUA}, compile) embed-lua: @@ -95,6 +94,6 @@ ed25519-donna: tinycc-ccache: tinycc tinycc: $(info -- Building tinycc embedded C compiler) - cd ${pwd}/lib/tinycc && CC="${libcc_cc}" AR=${ar} CFLAGS="${cflags}" \ + cd ${pwd}/lib/tinycc && CC="${libtcc_cc}" AR=${ar} CFLAGS="${cflags}" \ LDFLAGS="${ldflags}" ./configure --config-musl && \ $(MAKE) libtcc.a libtcc1.a diff --git a/build/init.mk b/build/init.mk index 2ea1ebfa3..62f5c6835 100644 --- a/build/init.mk +++ b/build/init.mk @@ -1,4 +1,4 @@ -BRANCH := $(shell git symbolic-ref HEAD | sed -e 's,.*/\(.*\),\1,') +BRANCH := $(shell git symbolic-ref HEAD | cut -d/ -f3-) COMMIT := $(shell git rev-parse --short HEAD) VERSION := $(shell git describe --tags | cut -d- -f1) CURRENT_YEAR := $(shell date +%Y) @@ -102,4 +102,5 @@ mimalloc_cflags += -fvisibility=hidden -Wstrict-prototypes mimalloc_cflags += -ftls-model=initial-exec -fno-builtin-malloc mimalloc_cflags += -DMI_USE_RTLGENRANDOM + # }}} diff --git a/build/linux.mk b/build/linux.mk index 8c4d668d7..a80bd6797 100644 --- a/build/linux.mk +++ b/build/linux.mk @@ -3,13 +3,6 @@ include build/init.mk COMPILER ?= gcc -cc := ${COMPILER} -quantum_proof_cc := ${cc} -ed25519_cc := ${cc} -libcc_cc := ${cc} -lua_cc := ${cc} -zenroom_cc := ${cc} - ## Additional dependencies BUILD_DEPS += tinycc mimalloc ldadd += -lm @@ -19,14 +12,6 @@ ldadd += ${pwd}/lib/mimalloc/build/libmimalloc-static.a cflags += -fPIC -D'ARCH="LINUX"' -DARCH_LINUX system := Linux -ifdef CCACHE - milagro_cmake_flags += -DCMAKE_C_COMPILER_LAUNCHER=ccache - quantum_proof_cc := ccache ${cc} - ed25519_cc := ccache ${cc} - libcc_cc := ccache ${cc} - lua_cc := ccache ${cc} - zenroom_cc := ccache ${cc} -endif # default is DEBUG ifdef RELEASE cflags += -O3 ${cflags_protection} @@ -34,7 +19,12 @@ else cflags += ${cflags_debug} endif -all: ${BUILD_DEPS} zenroom zencode-exec zencc +# activate CCACHE etc. +include build/plugins.mk + +all: deps zenroom zencode-exec zencc + +deps: ${BUILD_DEPS} cli_sources := src/cli-zenroom.o src/repl.o zenroom: ${ZEN_SOURCES} ${cli_sources} diff --git a/build/meson.mk b/build/meson.mk index 4090ecb1d..fec6d87fa 100644 --- a/build/meson.mk +++ b/build/meson.mk @@ -3,28 +3,18 @@ include build/init.mk BUILD_DEPS += mimalloc BUILD_DEPS += tinycc + +COMPILER ?= gcc + ## Specific compiler settings for all built dependencies ifdef RELEASE cflags += -O3 ${cflags_protection} -fPIC else cflags += ${cflags_debug} -fPIC endif -ifdef CLANG - cc := clang - zenroom_cc := ${cc} - quantum_proof_cc := ${cc} - ed25519_cc := ${cc} - lua_cc := ${cc} -endif -ifdef CCACHE - milagro_cmake_flags += -DCMAKE_C_COMPILER_LAUNCHER=ccache - mimalloc_cmake_flags += -DCMAKE_C_COMPILER_LAUNCHER=ccache - mimalloc_cmake_flags += -DCMAKE_CXX_COMPILER_LAUNCHER=ccache - zenroom_cc := ccache ${cc} - quantum_proof_cc := ccache ${cc} - ed25519_cc := ccache ${cc} - lua_cc := ccache ${cc} -endif + +# activate CCACHE etc. +include build/plugins.mk # MAIN TARGETS all: ${BUILD_DEPS} prepare diff --git a/build/musl-linux.mk b/build/musl-linux.mk index 3d2072a7d..273f82c61 100644 --- a/build/musl-linux.mk +++ b/build/musl-linux.mk @@ -1,12 +1,7 @@ ## Initialize build defaults include build/init.mk -cc := musl-gcc -quantum_proof_cc := ${cc} -ed25519_cc := ${cc} -libcc_cc := ${cc} -lua_cc := ${cc} -zenroom_cc := ${cc} +COMPILER := musl-gcc ## Additional dependencies BUILD_DEPS += tinycc @@ -16,20 +11,15 @@ cflags += -static -std=gnu99 -fPIC -D'ARCH="MUSL"' -D__MUSL__ -DARCH_MUSL ldflags += -static system := Linux -ifdef CCACHE - milagro_cmake_flags += -DCMAKE_C_COMPILER_LAUNCHER=ccache - quantum_proof_cc := ccache ${cc} - ed25519_cc := ccache ${cc} - libcc_cc := ccache ${cc} - lua_cc := ccache ${cc} - zenroom_cc := ccache ${cc} -endif ifdef DEBUG cflags += ${cflags_debug} else cflags += -O3 ${cflags_protection} endif +# activate CCACHE etc. +include build/plugins.mk + all: ${BUILD_DEPS} zenroom zencode-exec zencc cli_sources := src/cli-zenroom.o src/repl.o diff --git a/build/plugins.mk b/build/plugins.mk new file mode 100644 index 000000000..747319b29 --- /dev/null +++ b/build/plugins.mk @@ -0,0 +1,20 @@ +COMPILER ?= gcc + +cc := ${COMPILER} + +quantum_proof_cc := ${cc} +zenroom_cc := ${cc} +ed25519_cc := ${cc} +libtcc_cc := ${cc} +lua_cc := ${cc} + +ifdef CCACHE + milagro_cmake_flags += -DCMAKE_C_COMPILER_LAUNCHER=ccache + mimalloc_cmake_flags += -DCMAKE_C_COMPILER_LAUNCHER=ccache + mimalloc_cmake_flags += -DCMAKE_CXX_COMPILER_LAUNCHER=ccache + quantum_proof_cc := ccache ${cc} + zenroom_cc := ccache ${cc} + ed25519_cc := ccache ${cc} + libtcc_cc := ccache ${cc} + lua_cc := ccache ${cc} +endif diff --git a/build/win-exe.mk b/build/win-exe.mk index f4817dbc1..c777eb4b5 100644 --- a/build/win-exe.mk +++ b/build/win-exe.mk @@ -3,7 +3,7 @@ include build/init.mk BUILD_DEPS += mimalloc -cc := $(shell which x86_64-w64-mingw32-gcc) +COMPILER := $(shell which x86_64-w64-mingw32-gcc) ar := $(shell which x86_64-w64-mingw32-ar) ranlib := $(shell which x86_64-w64-mingw32-ranlib) ld := $(shell which x86_64-w64-mingw32-ld) @@ -18,17 +18,9 @@ ifdef RELEASE else cflags += ${cflags_debug} -fPIC endif -ifdef CCACHE - milagro_cmake_flags += -DCMAKE_C_COMPILER_LAUNCHER=ccache - mimalloc_cmake_flags += -DCMAKE_C_COMPILER_LAUNCHER=ccache - mimalloc_cmake_flags += -DCMAKE_CXX_COMPILER_LAUNCHER=ccache - zenroom_cc := ccache ${cc} - quantum_proof_cc := ccache ${cc} - ed25519_cc := ccache ${cc} - libcc_cc := ccache ${cc} - lua_cc := ccache ${cc} -endif +# activate CCACHE etc. +include build/plugins.mk all: ${BUILD_DEPS} stamp-exe-windres zenroom.exe zencode-exec.exe diff --git a/lib/pqclean/Makefile b/lib/pqclean/Makefile index 4e70c9ce8..f54154cdb 100644 --- a/lib/pqclean/Makefile +++ b/lib/pqclean/Makefile @@ -1,3 +1,6 @@ +# TODO: duplicate member names (cbd, symmetric shake) across ML- and others +# giving a warning on iOS, presumably can ignore as only on filename + CFLAGS ?= -O2 -I../../src -I. -fstack-protector-all -D_FORTIFY_SOURCE=2 -fno-strict-overflow CC ?= gcc