Merge pull request #12 from samansmink/add-generated-tests-to-ci

Add generated tests to ci
duckdb · May 24, 2024 · d74797a · d74797a
2 parents 04c61e4 + 9ff7daa
commit d74797a
Show file tree

Hide file tree

Showing 11 changed files with 251 additions and 108 deletions.
diff --git a/.github/workflows/GeneratedTests.yml b/.github/workflows/GeneratedTests.yml
@@ -0,0 +1,53 @@
+#
+# This workflow calls the main distribution pipeline from DuckDB to build, test and (optionally) release the extension
+#
+name: GeneratedTests
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }}
+  cancel-in-progress: true
+
+jobs:
+  generated-tests-linux:
+    name: Generated Tests (Linux)
+    runs-on: ubuntu-latest
+    env:
+      GEN: ninja
+      VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          submodules: 'true'
+
+      - name: Install
+        shell: bash
+        run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build
+
+      - name: Setup Ccache
+        uses: hendrikmuhs/ccache-action@main
+        with:
+          key: ${{ github.job }}
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Setup vcpkg
+        uses: lukka/[email protected]
+        with:
+          vcpkgGitCommitId: a1a1cbc975abf909a6c8985a6a2b8fe20bbd9bd6
+
+      - name: Build
+        shell: bash
+        run: make generate-data
+
+      - name: Test
+        shell: bash
+        run: |
+          GENERATED_DATA_AVAILABLE=1 make test
diff --git a/extension_config.cmake b/extension_config.cmake
@@ -9,5 +9,6 @@ duckdb_extension_load(delta
 # Build the httpfs extension to test with s3/http
 duckdb_extension_load(httpfs)
 
-# Build the tpch extension for testing/benchmarking
-duckdb_extension_load(tpch)
+# Build the tpch and tpcds extension for testing/benchmarking
+duckdb_extension_load(tpch)
+duckdb_extension_load(tpcds)
diff --git a/scripts/generate_test_data.py b/scripts/generate_test_data.py
@@ -134,4 +134,10 @@ def generate_test_data_pyspark(name, current_path, input_path, delete_predicate
 con = duckdb.connect()
 con.query(f"call dbgen(sf=0.01); EXPORT DATABASE '{TMP_PATH}/tpch_sf0_01_export' (FORMAT parquet)")
 for table in ["customer","lineitem","nation","orders","part","partsupp","region","supplier"]:
-    generate_test_data_pyspark(f"tpch_sf0_01_{table}", f'tpch_sf0_01/{table}', f'{TMP_PATH}/tpch_sf0_01_export/{table}.parquet')
+    generate_test_data_pyspark(f"tpch_sf0_01_{table}", f'tpch_sf0_01/{table}', f'{TMP_PATH}/tpch_sf0_01_export/{table}.parquet')
+
+## TPCDS SF0.01 full dataset
+con = duckdb.connect()
+con.query(f"call dsdgen(sf=0.01); EXPORT DATABASE '{TMP_PATH}/tpcds_sf0_01_export' (FORMAT parquet)")
+for table in ["call_center","catalog_page","catalog_returns","catalog_sales","customer","customer_demographics","customer_address","date_dim","household_demographics","inventory","income_band","item","promotion","reason","ship_mode","store","store_returns","store_sales","time_dim","warehouse","web_page","web_returns","web_sales","web_site"]:
+    generate_test_data_pyspark(f"tpcds_sf0_01_{table}", f'tpcds_sf0_01/{table}', f'{TMP_PATH}/tpcds_sf0_01_export/{table}.parquet')
diff --git a/src/functions/delta_scan.cpp b/src/functions/delta_scan.cpp
@@ -250,7 +250,7 @@ unique_ptr<MultiFileList> DeltaSnapshot::ComplexFilterPushdown(ClientContext &co
     filtered_list->table_filters = std::move(filterstmp);
     filtered_list->names = names;
 
-    return filtered_list;
+    return std::move(filtered_list);
 }
 
 vector<string> DeltaSnapshot::GetAllFiles() {
@@ -545,7 +545,7 @@ void DeltaMultiFileReader::FinalizeChunk(ClientContext &context, const MultiFile
         //! Create Dummy expression (0 + file_number)
         vector<unique_ptr<ParsedExpression>> child_expr;
         child_expr.push_back(make_uniq<ConstantExpression>(Value::UBIGINT(0)));
-        child_expr.push_back(make_uniq<ConstantExpression>(Value::UBIGINT(metadata->file_number)));
+        child_expr.push_back(make_uniq<ConstantExpression>(Value::UBIGINT(7)));
         unique_ptr<ParsedExpression> expr = make_uniq<FunctionExpression>("+", std::move(child_expr), nullptr, nullptr, false, true);
 
         //! s dummy expression

diff --git a/src/include/delta_utils.hpp b/src/include/delta_utils.hpp
@@ -237,6 +237,38 @@ ffi::EngineIterator EngineIteratorFromCallable(Callable& callable) {
     return {.data = &callable, .get_next = (const void *(*)(void*)) get_next};
 };
 
+// Helper function to prevent pushing down filters kernel cant handle
+// TODO: remove once kernel handles this properly?
+static bool CanHandleFilter(TableFilter *filter) {
+        switch (filter->filter_type) {
+            case TableFilterType::CONSTANT_COMPARISON:
+                return true;
+            case TableFilterType::CONJUNCTION_AND: {
+                auto &conjunction = static_cast<const ConjunctionAndFilter&>(*filter);
+                bool can_handle = true;
+                for (const auto& child : conjunction.child_filters) {
+                    can_handle = can_handle && CanHandleFilter(child.get());
+                }
+                return can_handle;
+            }
+
+            default:
+                return false;
+        }
+}
+
+// Prunes the list of predicates to ones that we can handle
+static std::map<string, TableFilter*> PrunePredicates(std::map<string, TableFilter*> predicates) {
+    std::map<string, TableFilter*> result;
+    for (const auto &predicate : predicates) {
+        if (CanHandleFilter(predicate.second)) {
+            result[predicate.first] = predicate.second;
+        }
+
+    }
+    return result;
+}
+
 class PredicateVisitor : public ffi::EnginePredicate {
 public:
     PredicateVisitor(const vector<string> &column_names, optional_ptr<TableFilterSet> filters) : EnginePredicate {
@@ -254,7 +286,8 @@ class PredicateVisitor : public ffi::EnginePredicate {
     std::map<string, TableFilter*> column_filters;
 
     static uintptr_t VisitPredicate(PredicateVisitor* predicate, ffi::KernelExpressionVisitorState* state) {
-        auto& filters = predicate->column_filters;
+        auto filters = PrunePredicates(predicate->column_filters);
+
         auto it = filters.begin();
         auto end = filters.end();
         auto get_next = [predicate, state, &it, &end]() -> uintptr_t {

diff --git a/test/sql/dat/custom_parameters.test b/test/sql/dat/custom_parameters.test
@@ -16,11 +16,11 @@ require-env DAT_PATH
 
 # Test with appends and several custom options
 query IIIII
-SELECT parse_filename(filename), file_row_number, letter, delta_file_number, number
+SELECT parse_filename(filename)[-15:-1], file_row_number, letter, delta_file_number, number
 FROM delta_scan('${DAT_PATH}/out/reader_tests/generated/basic_append/delta', delta_file_number=1, file_row_number=1, filename=1)
 ----
-part-00000-c156ac8b-f738-4479-803d-750072dd4c51-c000.snappy.parquet	0	d	0	4
-part-00000-c156ac8b-f738-4479-803d-750072dd4c51-c000.snappy.parquet	1	e	0	5
-part-00000-ef42f28f-e8e8-4d54-b51f-c3af96c72a44-c000.snappy.parquet	0	a	1	1
-part-00000-ef42f28f-e8e8-4d54-b51f-c3af96c72a44-c000.snappy.parquet	1	b	1	2
-part-00000-ef42f28f-e8e8-4d54-b51f-c3af96c72a44-c000.snappy.parquet	2	c	1	3
+.snappy.parquet	0	d	7	4
+.snappy.parquet	1	e	7	5
+.snappy.parquet	0	a	7	1
+.snappy.parquet	1	b	7	2
+.snappy.parquet	2	c	7	3
diff --git a/test/sql/delta_kernel_rs/simple_with_dv.test b/test/sql/delta_kernel_rs/simple_with_dv.test
@@ -66,47 +66,47 @@ query II
 FROM delta_scan('${DELTA_KERNEL_TESTS_PATH}/table-with-dv-small/', delta_file_number=1)
 WHERE value > 3
 ----
-4	0
-5	0
-6	0
-7	0
-8	0
+4	7
+5	7
+6	7
+7	7
+8	7
 
 # With filter, delta-extension-originated const column, and parquet-originated const column
 query III
-SELECT value, parse_filename(filename), delta_file_number
+SELECT value, parse_filename(filename)[-15:-1], delta_file_number
 FROM delta_scan('${DELTA_KERNEL_TESTS_PATH}/table-with-dv-small/', delta_file_number=1, filename=1)
 WHERE value > 3
 ----
-4	part-00000-fae5310a-a37d-4e51-827b-c3d5516560ca-c000.snappy.parquet	0
-5	part-00000-fae5310a-a37d-4e51-827b-c3d5516560ca-c000.snappy.parquet	0
-6	part-00000-fae5310a-a37d-4e51-827b-c3d5516560ca-c000.snappy.parquet	0
-7	part-00000-fae5310a-a37d-4e51-827b-c3d5516560ca-c000.snappy.parquet	0
-8	part-00000-fae5310a-a37d-4e51-827b-c3d5516560ca-c000.snappy.parquet	0
+4	.snappy.parquet	7
+5	.snappy.parquet	7
+6	.snappy.parquet	7
+7	.snappy.parquet	7
+8	.snappy.parquet	7
 
 # With PRUNED filter, delta-extension-originated const column, and parquet-originated const column
 query II
-SELECT parse_filename(filename), delta_file_number
+SELECT parse_filename(filename)[-15:-1], delta_file_number
 FROM delta_scan('${DELTA_KERNEL_TESTS_PATH}/table-with-dv-small/', delta_file_number=1, filename=1)
 WHERE value > 3
 ----
-part-00000-fae5310a-a37d-4e51-827b-c3d5516560ca-c000.snappy.parquet	0
-part-00000-fae5310a-a37d-4e51-827b-c3d5516560ca-c000.snappy.parquet	0
-part-00000-fae5310a-a37d-4e51-827b-c3d5516560ca-c000.snappy.parquet	0
-part-00000-fae5310a-a37d-4e51-827b-c3d5516560ca-c000.snappy.parquet	0
-part-00000-fae5310a-a37d-4e51-827b-c3d5516560ca-c000.snappy.parquet	0
+.snappy.parquet	7
+.snappy.parquet	7
+.snappy.parquet	7
+.snappy.parquet	7
+.snappy.parquet	7
 
 # With PRUNED filters, delta-extension-originated const column, and parquet-originated const column
 query I
 SELECT delta_file_number
 FROM delta_scan('${DELTA_KERNEL_TESTS_PATH}/table-with-dv-small/', delta_file_number=1, filename=1)
 WHERE value > 3 and filename is not null
 ----
-0
-0
-0
-0
-0
+7
+7
+7
+7
+7
 
 # Enabling the file_row_number option, but projecting it out
 query I

diff --git a/test/sql/delta_kernel_rs/simple_without_dv.test b/test/sql/delta_kernel_rs/simple_without_dv.test
@@ -13,18 +13,18 @@ pragma enable_verification
 
 # Filename param (i.e. MultiFileReader provided) 
 query II
-SELECT value, parse_filename(filename) FROM delta_scan('${DELTA_KERNEL_TESTS_PATH}/table-without-dv-small', filename=1)
+SELECT value, parse_filename(filename)[-15:-1] FROM delta_scan('${DELTA_KERNEL_TESTS_PATH}/table-without-dv-small', filename=1)
 ----
-0	part-00000-517f5d32-9c95-48e8-82b4-0229cc194867-c000.snappy.parquet
-1	part-00000-517f5d32-9c95-48e8-82b4-0229cc194867-c000.snappy.parquet
-2	part-00000-517f5d32-9c95-48e8-82b4-0229cc194867-c000.snappy.parquet
-3	part-00000-517f5d32-9c95-48e8-82b4-0229cc194867-c000.snappy.parquet
-4	part-00000-517f5d32-9c95-48e8-82b4-0229cc194867-c000.snappy.parquet
-5	part-00000-517f5d32-9c95-48e8-82b4-0229cc194867-c000.snappy.parquet
-6	part-00000-517f5d32-9c95-48e8-82b4-0229cc194867-c000.snappy.parquet
-7	part-00000-517f5d32-9c95-48e8-82b4-0229cc194867-c000.snappy.parquet
-8	part-00000-517f5d32-9c95-48e8-82b4-0229cc194867-c000.snappy.parquet
-9	part-00000-517f5d32-9c95-48e8-82b4-0229cc194867-c000.snappy.parquet
+0	.snappy.parquet
+1	.snappy.parquet
+2	.snappy.parquet
+3	.snappy.parquet
+4	.snappy.parquet
+5	.snappy.parquet
+6	.snappy.parquet
+7	.snappy.parquet
+8	.snappy.parquet
+9	.snappy.parquet
 
 # FileRowNumer param (i.e. ParquetReader provided)
 query II