diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml index 1ead9b1..ecc7a59 100644 --- a/.github/workflows/MainDistributionPipeline.yml +++ b/.github/workflows/MainDistributionPipeline.yml @@ -14,19 +14,21 @@ concurrency: jobs: duckdb-stable-build: name: Build extension binaries - uses: duckdb/duckdb/.github/workflows/_extension_distribution.yml@6812703823d1d66566bc7eaac2b6e4b273c85333 + uses: duckdb/duckdb/.github/workflows/_extension_distribution.yml@a056c8e0a29f2f38d245afeb9e48cc308711aa21 with: vcpkg_commit: a42af01b72c28a8e1d7b48107b33e4f286a55ef6 - duckdb_version: v0.9.2 + duckdb_version: a056c8e0a2 extension_name: sqlite_scanner + exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads' duckdb-stable-deploy: name: Deploy extension binaries needs: duckdb-stable-build - uses: ./.github/workflows/_extension_deploy.yml + uses: duckdb/duckdb/.github/workflows/_extension_deploy.yml@a056c8e0a29f2f38d245afeb9e48cc308711aa21 secrets: inherit with: - duckdb_version: v0.9.2 + duckdb_version: a056c8e0a2 extension_name: sqlite_scanner + exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads' deploy_latest: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }} deploy_versioned: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }} diff --git a/.github/workflows/_extension_deploy.yml b/.github/workflows/_extension_deploy.yml deleted file mode 100644 index 53493dc..0000000 --- a/.github/workflows/_extension_deploy.yml +++ /dev/null @@ -1,123 +0,0 @@ -# -# Reusable workflow that deploys the artifacts produced by github.com/duckdb/duckdb/.github/workflows/_extension_distribution.yml -# -# note: this workflow needs to be located in the extension repository, as it requires secrets to be passed to the -# deploy script. However, it should generally not be necessary to modify this workflow in your extension repository, as -# this workflow can be configured to use a custom deploy script. - - -name: Extension Deployment -on: - workflow_call: - inputs: - # The name of the extension - extension_name: - required: true - type: string - # DuckDB version to build against - duckdb_version: - required: true - type: string - # ';' separated list of architectures to exclude, for example: 'linux_amd64;osx_arm64' - exclude_archs: - required: false - type: string - default: "" - # Whether to upload this deployment as the latest. This may overwrite a previous deployment. - deploy_latest: - required: false - type: boolean - default: false - # Whether to upload this deployment under a versioned path. These will not be deleted automatically - deploy_versioned: - required: false - type: boolean - default: false - # Postfix added to artifact names. Can be used to guarantee unique names when this workflow is called multiple times - artifact_postfix: - required: false - type: string - default: "" - # Override the default deploy script with a custom script - deploy_script: - required: false - type: string - default: "./scripts/extension-upload.sh" - # Override the default matrix parse script with a custom script - matrix_parse_script: - required: false - type: string - default: "./duckdb/scripts/modify_distribution_matrix.py" - -jobs: - generate_matrix: - name: Generate matrix - runs-on: ubuntu-latest - outputs: - deploy_matrix: ${{ steps.parse-matrices.outputs.deploy_matrix }} - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: 'true' - - - name: Checkout DuckDB to version - run: | - cd duckdb - git checkout ${{ inputs.duckdb_version }} - - - id: parse-matrices - run: | - cat ./duckdb/.github/config/distribution_matrix.json > distribution_matrix.json - grep wasm distribution_matrix.json || (head -n -1 ./duckdb/.github/config/distribution_matrix.json > distribution_matrix.json && echo ',"wasm":{"include":[{"duckdb_arch":"wasm_mvp","vcpkg_triplet":"wasm32-emscripten"},{"duckdb_arch":"wasm_eh","vcpkg_triplet":"wasm32-emscripten"},{"duckdb_arch":"wasm_threads","vcpkg_triplet":"wasm32-emscripten"}]}}' >> distribution_matrix.json) - python3 ${{ inputs.matrix_parse_script }} --input distribution_matrix.json --deploy_matrix --output deploy_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty - deploy_matrix="`cat deploy_matrix.json`" - echo deploy_matrix=$deploy_matrix >> $GITHUB_OUTPUT - echo `cat $GITHUB_OUTPUT` - - deploy: - name: Deploy - runs-on: ubuntu-latest - needs: generate_matrix - if: ${{ needs.generate_matrix.outputs.deploy_matrix != '{}' && needs.generate_matrix.outputs.deploy_matrix != '' }} - strategy: - matrix: ${{fromJson(needs.generate_matrix.outputs.deploy_matrix)}} - - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: 'true' - - - name: Checkout DuckDB to version - run: | - cd duckdb - git checkout ${{ inputs.duckdb_version }} - - - uses: actions/download-artifact@v2 - with: - name: ${{ inputs.extension_name }}-${{ inputs.duckdb_version }}-extension-${{matrix.duckdb_arch}}${{inputs.artifact_postfix}}${{startsWith(matrix.duckdb, 'wasm') && '.wasm' || ''}} - path: | - /tmp/extension - - - name: Deploy - shell: bash - env: - AWS_ACCESS_KEY_ID: ${{ secrets.S3_DUCKDB_ORG_DEPLOY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DUCKDB_ORG_DEPLOY_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.S3_DUCKDB_ORG_REGION }} - BUCKET_NAME: ${{ secrets.S3_DUCKDB_ORG_BUCKET }} - DUCKDB_EXTENSION_SIGNING_PK: ${{ secrets.S3_DUCKDB_ORG_EXTENSION_SIGNING_PK }} - run: | - pwd - python3 -m pip install pip awscli - git config --global --add safe.directory '*' - cd duckdb - git fetch --tags - export DUCKDB_VERSION=`git tag --points-at HEAD` - export DUCKDB_VERSION=${DUCKDB_VERSION:=`git log -1 --format=%h`} - cd .. - git fetch --tags - export EXT_VERSION=`git tag --points-at HEAD` - export EXT_VERSION=${EXT_VERSION:=`git log -1 --format=%h`} - ${{ inputs.deploy_script }} ${{ inputs.extension_name }} $EXT_VERSION $DUCKDB_VERSION ${{ matrix.duckdb_arch }} $BUCKET_NAME ${{inputs.deploy_latest || 'true' && 'false'}} ${{inputs.deploy_versioned || 'true' && 'false'}} diff --git a/.gitmodules b/.gitmodules index 1b09733..bb8e599 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ [submodule "duckdb"] path = duckdb url = https://github.com/duckdb/duckdb - branch = master + branch = main diff --git a/Makefile b/Makefile index 93204f2..ce6dabc 100644 --- a/Makefile +++ b/Makefile @@ -5,12 +5,15 @@ all: release MKFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST))) PROJ_DIR := $(dir $(MKFILE_PATH)) +TEST_PATH="/test/unittest" +DUCKDB_PATH="/duckdb" + +# For non-MinGW windows the path is slightly different ifeq ($(OS),Windows_NT) +ifneq ($(CXX),g++) TEST_PATH="/test/Release/unittest.exe" DUCKDB_PATH="/Release/duckdb.exe" -else - TEST_PATH="/test/unittest" - DUCKDB_PATH="/duckdb" +endif endif OSX_ARCH_FLAG= @@ -59,7 +62,7 @@ release: data/db/tpch.db: release command -v sqlite3 || (command -v brew && brew install sqlite) || (command -v choco && choco install sqlite -y) || (command -v apt-get && apt-get install -y sqlite3) || echo "no sqlite3" - ./build/release/$(DUCKDB_PATH) < data/sql/tpch-export.duckdb + ./build/release/$(DUCKDB_PATH) < data/sql/tpch-export.duckdb || tree ./build/release || echo "neither tree not duck" sqlite3 data/db/tpch.db < data/sql/tpch-create.sqlite # Main tests diff --git a/duckdb b/duckdb index 401c806..a056c8e 160000 --- a/duckdb +++ b/duckdb @@ -1 +1 @@ -Subproject commit 401c8061c6ece35949cac58c7770cc755710ca86 +Subproject commit a056c8e0a29f2f38d245afeb9e48cc308711aa21 diff --git a/scripts/extension-upload.sh b/scripts/extension-upload.sh deleted file mode 100755 index 32097f2..0000000 --- a/scripts/extension-upload.sh +++ /dev/null @@ -1,90 +0,0 @@ -#!/bin/bash - -# Extension upload script - -# Usage: ./extension-upload.sh -# : Name of the extension -# : Version (commit / version tag) of the extension -# : Version (commit / version tag) of DuckDB -# : Architecture target of the extension binary -# : S3 bucket to upload to -# : Set this as the latest version ("true" / "false", default: "false") -# : Set this as a versioned version that will prevent its deletion - -set -e - -if [[ $4 == wasm* ]]; then - ext="/tmp/extension/$1.duckdb_extension.wasm" -else - ext="/tmp/extension/$1.duckdb_extension" -fi - -echo $ext - -script_dir="$(dirname "$(readlink -f "$0")")" - -# calculate SHA256 hash of extension binary -cat $ext > $ext.append - -if [[ $4 == wasm* ]]; then - # 0 for custom section - # 113 in hex = 275 in decimal, total lenght of what follows (1 + 16 + 2 + 256) - # [1(continuation) + 0010011(payload) = \x93, 0(continuation) + 10(payload) = \x02] - echo -n -e '\x00' >> $ext.append - echo -n -e '\x93\x02' >> $ext.append - # 10 in hex = 16 in decimal, lenght of name, 1 byte - echo -n -e '\x10' >> $ext.append - echo -n -e 'duckdb_signature' >> $ext.append - # the name of the WebAssembly custom section, 16 bytes - # 100 in hex, 256 in decimal - # [1(continuation) + 0000000(payload) = ff, 0(continuation) + 10(payload)], - # for a grand total of 2 bytes - echo -n -e '\x80\x02' >> $ext.append -fi - -# (Optionally) Sign binary -if [ "$DUCKDB_EXTENSION_SIGNING_PK" != "" ]; then - echo "$DUCKDB_EXTENSION_SIGNING_PK" > private.pem - $script_dir/../duckdb/scripts/compute-extension-hash.sh $ext.append > $ext.hash - openssl pkeyutl -sign -in $ext.hash -inkey private.pem -pkeyopt digest:sha256 -out $ext.sign - rm -f private.pem -fi - -# Signature is always there, potentially defaulting to 256 zeros -truncate -s 256 $ext.sign - -# append signature to extension binary -cat $ext.sign >> $ext.append - -# compress extension binary -if [[ $4 == wasm_* ]]; then - gzip < $ext.append > "$ext.compressed" -else - brotli < $ext.append > "$ext.compressed" -fi - -set -e - -# Abort if AWS key is not set -if [ -z "$AWS_ACCESS_KEY_ID" ]; then - echo "No AWS key found, skipping.." - exit 0 -fi - -# upload versioned version -if [[ $7 = 'true' ]]; then - if [[ $4 == wasm* ]]; then - aws s3 cp $ext.compressed s3://$5/duckdb-wasm/$1/$2/duckdb-wasm/$3/$4/$1.duckdb_extension.wasm --acl public-read --content-encoding br --content-type="application/wasm" - else - aws s3 cp $ext.compressed s3://$5/$1/$2/$3/$4/$1.duckdb_extension.gz --acl public-read - fi -fi - -# upload to latest version -if [[ $6 = 'true' ]]; then - if [[ $4 == wasm* ]]; then - aws s3 cp $ext.compressed s3://$5/duckdb-wasm/$3/$4/$1.duckdb_extension.wasm --acl public-read --content-encoding br --content-type="application/wasm" - else - aws s3 cp $ext.compressed s3://$5/$3/$4/$1.duckdb_extension.gz --acl public-read - fi -fi diff --git a/src/include/storage/sqlite_transaction_manager.hpp b/src/include/storage/sqlite_transaction_manager.hpp index 86b7d3c..4982eef 100644 --- a/src/include/storage/sqlite_transaction_manager.hpp +++ b/src/include/storage/sqlite_transaction_manager.hpp @@ -11,6 +11,7 @@ #include "duckdb/transaction/transaction_manager.hpp" #include "storage/sqlite_catalog.hpp" #include "storage/sqlite_transaction.hpp" +#include "duckdb/common/reference_map.hpp" namespace duckdb { @@ -18,16 +19,16 @@ class SQLiteTransactionManager : public TransactionManager { public: SQLiteTransactionManager(AttachedDatabase &db_p, SQLiteCatalog &sqlite_catalog); - Transaction *StartTransaction(ClientContext &context) override; - string CommitTransaction(ClientContext &context, Transaction *transaction) override; - void RollbackTransaction(Transaction *transaction) override; + Transaction &StartTransaction(ClientContext &context) override; + string CommitTransaction(ClientContext &context, Transaction &transaction) override; + void RollbackTransaction(Transaction &transaction) override; void Checkpoint(ClientContext &context, bool force = false) override; private: SQLiteCatalog &sqlite_catalog; mutex transaction_lock; - unordered_map> transactions; + reference_map_t> transactions; }; } // namespace duckdb diff --git a/src/sqlite_stmt.cpp b/src/sqlite_stmt.cpp index 158f2df..40d08cc 100644 --- a/src/sqlite_stmt.cpp +++ b/src/sqlite_stmt.cpp @@ -135,14 +135,14 @@ void SQLiteStatement::BindText(idx_t col, const string_t &value) { } template <> -void SQLiteStatement::Bind(idx_t col, nullptr_t value) { +void SQLiteStatement::Bind(idx_t col, std::nullptr_t value) { SQLiteUtils::Check(sqlite3_bind_null(stmt, col + 1), db); } void SQLiteStatement::BindValue(Vector &col, idx_t c, idx_t r) { auto &mask = FlatVector::Validity(col); if (!mask.RowIsValid(r)) { - Bind(c, nullptr); + Bind(c, nullptr); } else { switch (col.GetType().id()) { case LogicalTypeId::BIGINT: diff --git a/src/storage/sqlite_transaction_manager.cpp b/src/storage/sqlite_transaction_manager.cpp index 2b543ea..7b6e132 100644 --- a/src/storage/sqlite_transaction_manager.cpp +++ b/src/storage/sqlite_transaction_manager.cpp @@ -7,26 +7,26 @@ SQLiteTransactionManager::SQLiteTransactionManager(AttachedDatabase &db_p, SQLit : TransactionManager(db_p), sqlite_catalog(sqlite_catalog) { } -Transaction *SQLiteTransactionManager::StartTransaction(ClientContext &context) { +Transaction &SQLiteTransactionManager::StartTransaction(ClientContext &context) { auto transaction = make_uniq(sqlite_catalog, *this, context); transaction->Start(); - auto result = transaction.get(); + auto &result = *transaction; lock_guard l(transaction_lock); transactions[result] = std::move(transaction); return result; } -string SQLiteTransactionManager::CommitTransaction(ClientContext &context, Transaction *transaction) { - auto sqlite_transaction = (SQLiteTransaction *)transaction; - sqlite_transaction->Commit(); +string SQLiteTransactionManager::CommitTransaction(ClientContext &context, Transaction &transaction) { + auto &sqlite_transaction = transaction.Cast(); + sqlite_transaction.Commit(); lock_guard l(transaction_lock); transactions.erase(transaction); return string(); } -void SQLiteTransactionManager::RollbackTransaction(Transaction *transaction) { - auto sqlite_transaction = (SQLiteTransaction *)transaction; - sqlite_transaction->Rollback(); +void SQLiteTransactionManager::RollbackTransaction(Transaction &transaction) { + auto &sqlite_transaction = transaction.Cast(); + sqlite_transaction.Rollback(); lock_guard l(transaction_lock); transactions.erase(transaction); } diff --git a/test/sql/storage/attach_describe.test b/test/sql/storage/attach_describe.test index d356bef..b92ebd6 100644 --- a/test/sql/storage/attach_describe.test +++ b/test/sql/storage/attach_describe.test @@ -13,7 +13,7 @@ CREATE TABLE s1.test(i BIGINT PRIMARY KEY, j BIGINT DEFAULT 42); query IIIIII DESCRIBE s1.test ---- -i BIGINT YES NULL NULL NULL +i BIGINT YES PRI NULL NULL j BIGINT YES NULL 42 NULL query ITTTTT