From 645002db55fafc74a321986257ea3448db95fe3e Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Wed, 28 Feb 2024 12:59:00 +0100 Subject: [PATCH] split makefile to move it out --- Makefile | 152 +-------------------------------- extension_config.cmake | 11 +++ out-of-tree-extension.Makefile | 112 ++++++++++++++++++++++++ test/README.md | 13 ++- test/nodejs/quack_test.js | 31 ------- test/python/quack_test.py | 23 ----- 6 files changed, 131 insertions(+), 211 deletions(-) create mode 100644 extension_config.cmake create mode 100644 out-of-tree-extension.Makefile delete mode 100644 test/nodejs/quack_test.js delete mode 100644 test/python/quack_test.py diff --git a/Makefile b/Makefile index 0862e89..ddab169 100644 --- a/Makefile +++ b/Makefile @@ -1,152 +1,6 @@ -.PHONY: all clean format debug release duckdb_debug duckdb_release pull update wasm_mvp wasm_eh wasm_threads - -all: release - -MKFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST))) -PROJ_DIR := $(dir $(MKFILE_PATH)) - -TEST_PATH="/test/unittest" -DUCKDB_PATH="/duckdb" - -# For non-MinGW windows the path is slightly different -ifeq ($(OS),Windows_NT) -ifneq ($(CXX),g++) - TEST_PATH="/test/Release/unittest.exe" - DUCKDB_PATH="/Release/duckdb.exe" -endif -endif - -#### OSX config -OSX_BUILD_FLAG= -ifneq (${OSX_BUILD_ARCH}, "") - OSX_BUILD_FLAG=-DOSX_BUILD_ARCH=${OSX_BUILD_ARCH} -endif - -#### VCPKG config -VCPKG_TOOLCHAIN_PATH?= -ifneq ("${VCPKG_TOOLCHAIN_PATH}", "") - TOOLCHAIN_FLAGS:=${TOOLCHAIN_FLAGS} -DVCPKG_MANIFEST_DIR='${PROJ_DIR}' -DVCPKG_BUILD=1 -DCMAKE_TOOLCHAIN_FILE='${VCPKG_TOOLCHAIN_PATH}' -endif -ifneq ("${VCPKG_TARGET_TRIPLET}", "") - TOOLCHAIN_FLAGS:=${TOOLCHAIN_FLAGS} -DVCPKG_TARGET_TRIPLET='${VCPKG_TARGET_TRIPLET}' -endif - -#### Enable Ninja as generator -ifeq ($(GEN),ninja) - GENERATOR=-G "Ninja" -DFORCE_COLORED_OUTPUT=1 -endif +PROJ_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) EXT_NAME=quack +EXT_CONFIG=${PROJ_DIR}extension_config.cmake -#### Configuration for this extension -EXTENSION_NAME=QUACK -EXTENSION_FLAGS=\ --DDUCKDB_EXTENSION_NAMES="${EXT_NAME}" \ --DDUCKDB_EXTENSION_${EXTENSION_NAME}_PATH="$(PROJ_DIR)" \ --DDUCKDB_EXTENSION_${EXTENSION_NAME}_LOAD_TESTS=1 \ --DDUCKDB_EXTENSION_${EXTENSION_NAME}_INCLUDE_PATH="$(PROJ_DIR)src/include" \ --DDUCKDB_EXTENSION_${EXTENSION_NAME}_TEST_PATH="$(PROJ_DIR)test/sql" - -#### Add more of the DuckDB in-tree extensions here that you need (also feel free to remove them when not needed) -EXTRA_EXTENSIONS_FLAG=-DBUILD_EXTENSIONS="tpch" - -BUILD_FLAGS=-DEXTENSION_STATIC_BUILD=1 $(EXTENSION_FLAGS) ${EXTRA_EXTENSIONS_FLAG} $(OSX_BUILD_FLAG) $(TOOLCHAIN_FLAGS) -DDUCKDB_EXPLICIT_PLATFORM='${DUCKDB_PLATFORM}' -CLIENT_FLAGS:= - -#### Main build -# For regular CLI build, we link the quack extension directly into the DuckDB executable -CLIENT_FLAGS=-DDUCKDB_EXTENSION_${EXTENSION_NAME}_SHOULD_LINK=1 - -debug: - mkdir -p build/debug && \ - cmake $(GENERATOR) $(BUILD_FLAGS) $(CLIENT_FLAGS) -DCMAKE_BUILD_TYPE=Debug -S ./duckdb/ -B build/debug && \ - cmake --build build/debug --config Debug - -release: - mkdir -p build/release && \ - cmake $(GENERATOR) $(BUILD_FLAGS) $(CLIENT_FLAGS) -DCMAKE_BUILD_TYPE=Release -S ./duckdb/ -B build/release && \ - cmake --build build/release --config Release - -##### Client build -JS_BUILD_FLAGS=-DBUILD_NODE=1 -DDUCKDB_EXTENSION_${EXTENSION_NAME}_SHOULD_LINK=0 -PY_BUILD_FLAGS=-DBUILD_PYTHON=1 -DDUCKDB_EXTENSION_${EXTENSION_NAME}_SHOULD_LINK=0 - -debug_js: CLIENT_FLAGS=$(JS_BUILD_FLAGS) -debug_js: debug -debug_python: CLIENT_FLAGS=$(PY_BUILD_FLAGS) -debug_python: debug -release_js: CLIENT_FLAGS=$(JS_BUILD_FLAGS) -release_js: release -release_python: CLIENT_FLAGS=$(PY_BUILD_FLAGS) -release_python: release - -# Main tests -test: test_release - -test_release: release - ./build/release/$(TEST_PATH) "$(PROJ_DIR)test/*" - -test_debug: debug - ./build/debug/$(TEST_PATH) "$(PROJ_DIR)test/*" - -#### Client tests -DEBUG_EXT_PATH='$(PROJ_DIR)build/debug/extension/quack/quack.duckdb_extension' -RELEASE_EXT_PATH='$(PROJ_DIR)build/release/extension/quack/quack.duckdb_extension' - -test_js: test_debug_js - -test_debug_js: debug_js - cd duckdb/tools/nodejs && ${EXTENSION_NAME}_EXTENSION_BINARY_PATH=$(DEBUG_EXT_PATH) npm run test-path -- "../../../test/nodejs/**/*.js" - -test_release_js: release_js - cd duckdb/tools/nodejs && ${EXTENSION_NAME}_EXTENSION_BINARY_PATH=$(RELEASE_EXT_PATH) npm run test-path -- "../../../test/nodejs/**/*.js" - -test_python: test_debug_python - -test_debug_python: debug_python - cd test/python && ${EXTENSION_NAME}_EXTENSION_BINARY_PATH=$(DEBUG_EXT_PATH) python3 -m pytest - -test_release_python: release_python - cd test/python && ${EXTENSION_NAME}_EXTENSION_BINARY_PATH=$(RELEASE_EXT_PATH) python3 -m pytest - -#### Misc -format: - find src/ -iname *.hpp -o -iname *.cpp | xargs clang-format --sort-includes=0 -style=file -i - cmake-format -i CMakeLists.txt -update: - git submodule update --remote --merge -pull: - git submodule init - git submodule update --recursive --remote - -clean: - rm -rf build - rm -rf testext - cd duckdb && make clean - cd duckdb && make clean-python - - -VCPKG_EMSDK_FLAGS=-DVCPKG_CHAINLOAD_TOOLCHAIN_FILE=$(EMSDK)/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake -WASM_COMPILE_TIME_COMMON_FLAGS=-DWASM_LOADABLE_EXTENSIONS=1 -DBUILD_EXTENSIONS_ONLY=1 -DSKIP_EXTENSIONS="parquet;json" $(TOOLCHAIN_FLAGS) $(VCPKG_EMSDK_FLAGS) -DDUCKDB_CUSTOM_PLATFORM='${DUCKDB_PLATFORM}' -DDUCKDB_EXPLICIT_PLATFORM='${DUCKDB_PLATFORM}' -WASM_CXX_MVP_FLAGS= -WASM_CXX_EH_FLAGS=$(WASM_CXX_MVP_FLAGS) -fwasm-exceptions -DWEBDB_FAST_EXCEPTIONS=1 -WASM_CXX_THREADS_FLAGS=$(WASM_COMPILE_TIME_EH_FLAGS) -DWITH_WASM_THREADS=1 -DWITH_WASM_SIMD=1 -DWITH_WASM_BULK_MEMORY=1 -pthread -WASM_LINK_TIME_FLAGS=-O3 -sSIDE_MODULE=2 -sEXPORTED_FUNCTIONS="_${EXT_NAME}_version,_${EXT_NAME}_init" - -wasm_mvp: - mkdir -p build/wasm_mvp - emcmake cmake $(GENERATOR) $(EXTENSION_FLAGS) $(WASM_COMPILE_TIME_COMMON_FLAGS) -Bbuild/wasm_mvp -DCMAKE_CXX_FLAGS="$(WASM_CXX_MVP_FLAGS)" -S duckdb - emmake make -j8 -Cbuild/wasm_mvp - cd build/wasm_mvp/extension/${EXT_NAME} && emcc $f -o ../../${EXT_NAME}.duckdb_extension.wasm ${EXT_NAME}.duckdb_extension.wasm.lib $(WASM_LINK_TIME_FLAGS) - -wasm_eh: - mkdir -p build/wasm_eh - emcmake cmake $(GENERATOR) $(EXTENSION_FLAGS) $(WASM_COMPILE_TIME_COMMON_FLAGS) -Bbuild/wasm_eh -DCMAKE_CXX_FLAGS="$(WASM_CXX_EH_FLAGS)" -S duckdb - emmake make -j8 -Cbuild/wasm_eh - cd build/wasm_eh/extension/${EXT_NAME} && emcc $f -o ../../${EXT_NAME}.duckdb_extension.wasm ${EXT_NAME}.duckdb_extension.wasm.lib $(WASM_LINK_TIME_FLAGS) - -wasm_threads: - mkdir -p ./build/wasm_threads - emcmake cmake $(GENERATOR) $(EXTENSION_FLAGS) $(WASM_COMPILE_TIME_COMMON_FLAGS) -Bbuild/wasm_threads -DCMAKE_CXX_FLAGS="$(WASM_CXX_THREADS_FLAGS)" -S duckdb - emmake make -j8 -Cbuild/wasm_threads - cd build/wasm_threads/extension/${EXT_NAME} && emcc $f -o ../../${EXT_NAME}.duckdb_extension.wasm ${EXT_NAME}.duckdb_extension.wasm.lib $(WASM_LINK_TIME_FLAGS) -sSHARED_MEMORY=1 -pthread +include out-of-tree-extension.Makefile \ No newline at end of file diff --git a/extension_config.cmake b/extension_config.cmake new file mode 100644 index 0000000..ea06c11 --- /dev/null +++ b/extension_config.cmake @@ -0,0 +1,11 @@ +# Config to build this extension +duckdb_extension_load(quack + SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR} + INCLUDE_PATH ${CMAKE_CURRENT_LIST_DIR}src/include + TEST_PATH ${CMAKE_CURRENT_LIST_DIR}src/include + LOAD_TESTS + DONT_LINK +) + +# Load more extensions in your builds by adding them here. E.g.: +# duckdb_extension_load(json) \ No newline at end of file diff --git a/out-of-tree-extension.Makefile b/out-of-tree-extension.Makefile new file mode 100644 index 0000000..e8c0e7d --- /dev/null +++ b/out-of-tree-extension.Makefile @@ -0,0 +1,112 @@ +# Reusable makefile for building out-of-tree extension with the DuckDB extension template +# +# TODO: move out of this repo +# +# Inputs +# EXT_NAME : Upper case string describing the name of the out-of-tree extension +# EXT_CONFIG : Path to the extension config file specifying how to build the extension +# EXT_FLAGS : Extra CMake flags to pass to the build +# EXT_RELEASE_FLAGS : Extra CMake flags to pass to the release build +# EXT_DEBUG_FLAGS : Extra CMake flags to pass to the debug build + +.PHONY: all clean format debug release pull update wasm_mvp wasm_eh wasm_threads + +all: release + +TEST_PATH="/test/unittest" +DUCKDB_PATH="/duckdb" + +# For non-MinGW windows the path is slightly different +ifeq ($(OS),Windows_NT) +ifneq ($(CXX),g++) + TEST_PATH="/test/Release/unittest.exe" + DUCKDB_PATH="/Release/duckdb.exe" +endif +endif + +#### OSX config +OSX_BUILD_FLAG= +ifneq (${OSX_BUILD_ARCH}, "") + OSX_BUILD_FLAG=-DOSX_BUILD_ARCH=${OSX_BUILD_ARCH} +endif + +#### VCPKG config +VCPKG_TOOLCHAIN_PATH?= +ifneq ("${VCPKG_TOOLCHAIN_PATH}", "") + TOOLCHAIN_FLAGS:=${TOOLCHAIN_FLAGS} -DVCPKG_MANIFEST_DIR='${PROJ_DIR}' -DVCPKG_BUILD=1 -DCMAKE_TOOLCHAIN_FILE='${VCPKG_TOOLCHAIN_PATH}' +endif +ifneq ("${VCPKG_TARGET_TRIPLET}", "") + TOOLCHAIN_FLAGS:=${TOOLCHAIN_FLAGS} -DVCPKG_TARGET_TRIPLET='${VCPKG_TARGET_TRIPLET}' +endif + +#### Enable Ninja as generator +ifeq ($(GEN),ninja) + GENERATOR=-G "Ninja" -DFORCE_COLORED_OUTPUT=1 +endif + +#### Configuration for this extension +EXTENSION_FLAGS=-DDUCKDB_EXTENSION_CONFIGS='${EXT_CONFIG}' + +BUILD_FLAGS=-DEXTENSION_STATIC_BUILD=1 $(EXTENSION_FLAGS) ${EXT_FLAGS} $(OSX_BUILD_FLAG) $(TOOLCHAIN_FLAGS) -DDUCKDB_EXPLICIT_PLATFORM='${DUCKDB_PLATFORM}' + +debug: + mkdir -p build/debug && \ + cmake $(GENERATOR) $(BUILD_FLAGS) $(EXT_DEBUG_FLAGS) -DCMAKE_BUILD_TYPE=Debug -S ./duckdb/ -B build/debug && \ + cmake --build build/debug --config Debug + +release: + mkdir -p build/release && \ + cmake $(GENERATOR) $(BUILD_FLAGS) $(EXT_RELEASE_FLAGS) -DCMAKE_BUILD_TYPE=Release -S ./duckdb/ -B build/release && \ + cmake --build build/release --config Release + +# Main tests +test: test_release + +test_release: release + ./build/release/$(TEST_PATH) "$(PROJ_DIR)test/*" + +test_debug: debug + ./build/debug/$(TEST_PATH) "$(PROJ_DIR)test/*" + +# WASM config +VCPKG_EMSDK_FLAGS=-DVCPKG_CHAINLOAD_TOOLCHAIN_FILE=$(EMSDK)/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake +WASM_COMPILE_TIME_COMMON_FLAGS=-DWASM_LOADABLE_EXTENSIONS=1 -DBUILD_EXTENSIONS_ONLY=1 -DSKIP_EXTENSIONS="parquet;json" $(TOOLCHAIN_FLAGS) $(VCPKG_EMSDK_FLAGS) -DDUCKDB_CUSTOM_PLATFORM='${DUCKDB_PLATFORM}' -DDUCKDB_EXPLICIT_PLATFORM='${DUCKDB_PLATFORM}' +WASM_CXX_MVP_FLAGS= +WASM_CXX_EH_FLAGS=$(WASM_CXX_MVP_FLAGS) -fwasm-exceptions -DWEBDB_FAST_EXCEPTIONS=1 +WASM_CXX_THREADS_FLAGS=$(WASM_COMPILE_TIME_EH_FLAGS) -DWITH_WASM_THREADS=1 -DWITH_WASM_SIMD=1 -DWITH_WASM_BULK_MEMORY=1 -pthread +WASM_LINK_TIME_FLAGS=-O3 -sSIDE_MODULE=2 -sEXPORTED_FUNCTIONS="_${EXT_NAME}_version,_${EXT_NAME}_init" + +# WASM targets +wasm_mvp: + mkdir -p build/wasm_mvp + emcmake cmake $(GENERATOR) $(EXTENSION_FLAGS) $(WASM_COMPILE_TIME_COMMON_FLAGS) -Bbuild/wasm_mvp -DCMAKE_CXX_FLAGS="$(WASM_CXX_MVP_FLAGS)" -S duckdb + emmake make -j8 -Cbuild/wasm_mvp + cd build/wasm_mvp/extension/${EXT_NAME} && emcc $f -o ../../${EXT_NAME}.duckdb_extension.wasm ${EXT_NAME}.duckdb_extension.wasm.lib $(WASM_LINK_TIME_FLAGS) + +wasm_eh: + mkdir -p build/wasm_eh + emcmake cmake $(GENERATOR) $(EXTENSION_FLAGS) $(WASM_COMPILE_TIME_COMMON_FLAGS) -Bbuild/wasm_eh -DCMAKE_CXX_FLAGS="$(WASM_CXX_EH_FLAGS)" -S duckdb + emmake make -j8 -Cbuild/wasm_eh + cd build/wasm_eh/extension/${EXT_NAME} && emcc $f -o ../../${EXT_NAME}.duckdb_extension.wasm ${EXT_NAME}.duckdb_extension.wasm.lib $(WASM_LINK_TIME_FLAGS) + +wasm_threads: + mkdir -p ./build/wasm_threads + emcmake cmake $(GENERATOR) $(EXTENSION_FLAGS) $(WASM_COMPILE_TIME_COMMON_FLAGS) -Bbuild/wasm_threads -DCMAKE_CXX_FLAGS="$(WASM_CXX_THREADS_FLAGS)" -S duckdb + emmake make -j8 -Cbuild/wasm_threads + cd build/wasm_threads/extension/${EXT_NAME} && emcc $f -o ../../${EXT_NAME}.duckdb_extension.wasm ${EXT_NAME}.duckdb_extension.wasm.lib $(WASM_LINK_TIME_FLAGS) -sSHARED_MEMORY=1 -pthread + +#### Misc +format: + find src/ -iname *.hpp -o -iname *.cpp | xargs clang-format --sort-includes=0 -style=file -i + cmake-format -i CMakeLists.txt +update: + git submodule update --remote --merge +pull: + git submodule init + git submodule update --recursive --remote + +clean: + rm -rf build + rm -rf testext + cd duckdb && make clean + cd duckdb && make clean-python \ No newline at end of file diff --git a/test/README.md b/test/README.md index f2625b8..95e70ce 100644 --- a/test/README.md +++ b/test/README.md @@ -1,14 +1,11 @@ # Testing the quack extension -This directory contains all the tests for the quack extension. The `sql` directory holds tests that are written as [SQLLogicTests](https://duckdb.org/dev/sqllogictest/intro.html). DuckDB aims to have most its tests in this format as SQL statements, so for the quack extension, this should probably be the goal too. However, client specific testing is also available. +This directory contains all the tests for the quack extension. The `sql` directory holds tests that are written as [SQLLogicTests](https://duckdb.org/dev/sqllogictest/intro.html). DuckDB aims to have most its tests in this format as SQL statements, so for the quack extension, this should probably be the goal too. The root makefile contains targets to build and run all of these tests. To run the SQLLogicTests: ```bash make test ``` - -To run the python tests: -```sql -make test_python -``` - -For other client tests check the makefile in the root of this repository. \ No newline at end of file +or +```bash +make test_debug +``` \ No newline at end of file diff --git a/test/nodejs/quack_test.js b/test/nodejs/quack_test.js deleted file mode 100644 index b6b09c9..0000000 --- a/test/nodejs/quack_test.js +++ /dev/null @@ -1,31 +0,0 @@ -var duckdb = require('../../duckdb/tools/nodejs'); -var assert = require('assert'); - -describe(`quack extension`, () => { - let db; - let conn; - before((done) => { - db = new duckdb.Database(':memory:', {"allow_unsigned_extensions":"true"}); - conn = new duckdb.Connection(db); - conn.exec(`LOAD '${process.env.QUACK_EXTENSION_BINARY_PATH}';`, function (err) { - if (err) throw err; - done(); - }); - }); - - it('quack function should return expected string', function (done) { - db.all("SELECT quack('Sam') as value;", function (err, res) { - if (err) throw err; - assert.deepEqual(res, [{value: "Quack Sam 🐥"}]); - done(); - }); - }); - - it('quack_openssl_version function should return expected string', function (done) { - db.all("SELECT quack_openssl_version('Michael') as value;", function (err, res) { - if (err) throw err; - assert(res[0].value.startsWith('Quack Michael, my linked OpenSSL version is OpenSSL')); - done(); - }); - }); -}); \ No newline at end of file diff --git a/test/python/quack_test.py b/test/python/quack_test.py deleted file mode 100644 index e2be276..0000000 --- a/test/python/quack_test.py +++ /dev/null @@ -1,23 +0,0 @@ -import duckdb -import os -import pytest - -# Get a fresh connection to DuckDB with the quack extension binary loaded -@pytest.fixture -def duckdb_conn(): - extension_binary = os.getenv('QUACK_EXTENSION_BINARY_PATH') - if (extension_binary == ''): - raise Exception('Please make sure the `QUACK_EXTENSION_BINARY_PATH` is set to run the python tests') - conn = duckdb.connect('', config={'allow_unsigned_extensions': 'true'}) - conn.execute(f"load '{extension_binary}'") - return conn - -def test_quack(duckdb_conn): - duckdb_conn.execute("SELECT quack('Sam') as value;"); - res = duckdb_conn.fetchall() - assert res[0][0] == "Quack Sam 🐥" - -def test_quack_openssl_version_test(duckdb_conn): - duckdb_conn.execute("SELECT quack_openssl_version('Michael');"); - res = duckdb_conn.fetchall() - assert res[0][0][0:51] == "Quack Michael, my linked OpenSSL version is OpenSSL"