Merge branch 'master' into proj_optimization_analyzer

ClibMouse · May 13, 2024 · 812ea28 · 812ea28
2 parents 6c25e95 + 67cf408
commit 812ea28
Show file tree

Hide file tree

Showing 695 changed files with 3,806 additions and 2,088 deletions.
diff --git a/.clang-tidy b/.clang-tidy
@@ -123,7 +123,21 @@ Checks: [
   '-readability-uppercase-literal-suffix',
   '-readability-use-anyofallof',
 
-  '-zircon-*'
+  '-zircon-*',
+
+  # These are new in clang-18, and we have to sort them out:
+  '-readability-avoid-nested-conditional-operator',
+  '-modernize-use-designated-initializers',
+  '-performance-enum-size',
+  '-readability-redundant-inline-specifier',
+  '-readability-redundant-member-init',
+  '-bugprone-crtp-constructor-accessibility',
+  '-bugprone-suspicious-stringview-data-usage',
+  '-bugprone-multi-level-implicit-pointer-conversion',
+  '-cert-err33-c',
+
+  # This is a good check, but clang-tidy crashes, see https://github.com/llvm/llvm-project/issues/91872
+  '-modernize-use-constraints',
 ]
 
 WarningsAsErrors: '*'

diff --git a/.github/workflows/merge_queue.yml b/.github/workflows/merge_queue.yml
@@ -0,0 +1,96 @@
+# yamllint disable rule:comments-indentation
+name: MergeQueueCI
+
+env:
+  # Force the stdout and stderr streams to be unbuffered
+  PYTHONUNBUFFERED: 1
+
+on:  # yamllint disable-line rule:truthy
+  merge_group:
+
+jobs:
+  RunConfig:
+    runs-on: [self-hosted, style-checker-aarch64]
+    outputs:
+      data: ${{ steps.runconfig.outputs.CI_DATA }}
+    steps:
+      - name: DebugInfo
+        uses: hmarr/debug-action@f7318c783045ac39ed9bb497e22ce835fdafbfe6
+      - name: Check out repository code
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true # to ensure correct digests
+          fetch-depth: 0 # to get version
+          filter: tree:0
+      - name: Python unit tests
+        run: |
+          cd "$GITHUB_WORKSPACE/tests/ci"
+          echo "Testing the main ci directory"
+          python3 -m unittest discover -s . -p 'test_*.py'
+          for dir in *_lambda/; do
+            echo "Testing $dir"
+            python3 -m unittest discover -s "$dir" -p 'test_*.py'
+          done
+      - name: PrepareRunConfig
+        id: runconfig
+        run: |
+            python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --configure --outfile ${{ runner.temp }}/ci_run_data.json
+
+            echo "::group::CI configuration"
+            python3 -m json.tool ${{ runner.temp }}/ci_run_data.json
+            echo "::endgroup::"
+
+            {
+              echo 'CI_DATA<<EOF'
+              cat  ${{ runner.temp }}/ci_run_data.json
+              echo 'EOF'
+            } >> "$GITHUB_OUTPUT"
+  BuildDockers:
+    needs: [RunConfig]
+    if: ${{ !failure() && !cancelled() && toJson(fromJson(needs.RunConfig.outputs.data).docker_data.missing_multi) != '[]' }}
+    uses: ./.github/workflows/reusable_docker.yml
+    with:
+      data: ${{ needs.RunConfig.outputs.data }}
+  StyleCheck:
+    needs: [RunConfig, BuildDockers]
+    if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'Style check')}}
+    uses: ./.github/workflows/reusable_test.yml
+    with:
+      test_name: Style check
+      runner_type: style-checker
+      run_command: |
+          python3 style_check.py
+      data: ${{ needs.RunConfig.outputs.data }}
+    secrets:
+      secret_envs: |
+        ROBOT_CLICKHOUSE_SSH_KEY<<RCSK
+        ${{secrets.ROBOT_CLICKHOUSE_SSH_KEY}}
+        RCSK
+  FastTest:
+    needs: [RunConfig, BuildDockers]
+    if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'Fast test') }}
+    uses: ./.github/workflows/reusable_test.yml
+    with:
+      test_name: Fast test
+      runner_type: builder
+      data: ${{ needs.RunConfig.outputs.data }}
+      run_command: |
+          python3 fast_test_check.py
+
+  ################################# Stage Final #################################
+  #
+  FinishCheck:
+    if: ${{ !failure() && !cancelled() }}
+    needs: [RunConfig, BuildDockers, StyleCheck, FastTest]
+    runs-on: [self-hosted, style-checker]
+    steps:
+      - name: Check out repository code
+        uses: ClickHouse/checkout@v1
+      - name: Check sync status
+        run: |
+          cd "$GITHUB_WORKSPACE/tests/ci"
+          python3 sync_pr.py --status
+      - name: Finish label
+        run: |
+          cd "$GITHUB_WORKSPACE/tests/ci"
+          python3 finish_check.py ${{ (contains(needs.*.result, 'failure') && github.event_name == 'merge_group') && '--pipeline-failure' || '' }}
diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
@@ -6,7 +6,6 @@ env:
   PYTHONUNBUFFERED: 1
 
 on:  # yamllint disable-line rule:truthy
-  merge_group:
   pull_request:
     types:
       - synchronize
@@ -30,7 +29,6 @@ jobs:
           fetch-depth: 0 # to get version
           filter: tree:0
       - name: Labels check
-        if: ${{ github.event_name != 'merge_group' }}
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"
           python3 run_check.py
@@ -58,7 +56,6 @@ jobs:
               echo 'EOF'
             } >> "$GITHUB_OUTPUT"
       - name: Re-create GH statuses for skipped jobs if any
-        if: ${{ github.event_name != 'merge_group' }}
         run: |
             python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ runner.temp }}/ci_run_data.json --update-gh-statuses
   BuildDockers:
@@ -83,7 +80,7 @@ jobs:
         ${{secrets.ROBOT_CLICKHOUSE_SSH_KEY}}
         RCSK
   FastTest:
-    needs: [RunConfig, BuildDockers]
+    needs: [RunConfig, BuildDockers, StyleCheck]
     if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'Fast test') }}
     uses: ./.github/workflows/reusable_test.yml
     with:
@@ -163,20 +160,16 @@ jobs:
     steps:
       - name: Check out repository code
         uses: ClickHouse/checkout@v1
-      - name: Check sync status
-        if: ${{ github.event_name == 'merge_group' }}
-        run: |
-          cd "$GITHUB_WORKSPACE/tests/ci"
-          python3 sync_pr.py --status
       - name: Finish label
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"
-          python3 finish_check.py ${{ (contains(needs.*.result, 'failure') && github.event_name == 'merge_group') && '--pipeline-failure' || '' }}
-      - name: Auto merge if approved
-        if: ${{ github.event_name != 'merge_group' }}
-        run: |
-          cd "$GITHUB_WORKSPACE/tests/ci"
-          python3 merge_pr.py --check-approved
+          python3 finish_check.py
+      # FIXME: merge on approval does not work with MQ. Could be fixed by using defaul GH's automerge after some corrections in Mergeable Check status
+      # - name: Auto merge if approved
+      #   if: ${{ github.event_name != 'merge_group' }}
+      #   run: |
+      #     cd "$GITHUB_WORKSPACE/tests/ci"
+      #     python3 merge_pr.py --check-approved
 
 
 #############################################################################################

diff --git a/.gitmodules b/.gitmodules
@@ -6,7 +6,7 @@
 	url = https://github.com/facebook/zstd
 [submodule "contrib/lz4"]
 	path = contrib/lz4
-	url = https://github.com/ClickHouse/lz4
+	url = https://github.com/lz4/lz4
 [submodule "contrib/librdkafka"]
 	path = contrib/librdkafka
 	url = https://github.com/ClickHouse/librdkafka

diff --git a/README.md b/README.md
@@ -40,7 +40,6 @@ Every month we get together with the community (users, contributors, customers,
 
 Keep an eye out for upcoming meetups and events around the world. Somewhere else you want us to be? Please feel free to reach out to tyler `<at>` clickhouse `<dot>` com. You can also peruse [ClickHouse Events](https://clickhouse.com/company/news-events) for a list of all upcoming trainings, meetups, speaking engagements, etc.
 
-* [ClickHouse Meetup in Bengaluru](https://www.meetup.com/clickhouse-bangalore-user-group/events/300405581/) - May 4
 * [ClickHouse Happy Hour @ Tom's Watch Bar - Los Angeles](https://www.meetup.com/clickhouse-los-angeles-user-group/events/300740584/) - May 22
 * [ClickHouse & Confluent Meetup in Dubai](https://www.meetup.com/clickhouse-dubai-meetup-group/events/299629189/) - May 28
 * [ClickHouse Meetup in Stockholm](https://www.meetup.com/clickhouse-stockholm-user-group/events/299752651/) - Jun 3
@@ -49,6 +48,7 @@ Keep an eye out for upcoming meetups and events around the world. Somewhere else
 * [ClickHouse Meetup in Amsterdam](https://www.meetup.com/clickhouse-netherlands-user-group/events/300781068/) - Jun 27
 * [ClickHouse Meetup in Paris](https://www.meetup.com/clickhouse-france-user-group/events/300783448/) - Jul 9
 * [ClickHouse Meetup @ Ramp - New York City](https://www.meetup.com/clickhouse-new-york-user-group/events/300595845/) - Jul 9
+* [ClickHouse Meetup @ Klaviyo - Boston](https://www.meetup.com/clickhouse-boston-user-group/events/300907870) - Jul 11
 
 ## Recent Recordings
 * **Recent Meetup Videos**: [Meetup Playlist](https://www.youtube.com/playlist?list=PL0Z2YDlm0b3iNDUzpY1S3L_iV4nARda_U) Whenever possible recordings of the ClickHouse Community Meetups are edited and presented as individual talks. Current featuring "Modern SQL in 2023", "Fast, Concurrent, and Consistent Asynchronous INSERTS in ClickHouse", and "Full-Text Indices: Design and Experiments"

diff --git a/base/base/DecomposedFloat.h b/base/base/DecomposedFloat.h
@@ -51,11 +51,9 @@ struct DecomposedFloat
     /// Returns 0 for both +0. and -0.
     int sign() const
     {
-        return (exponent() == 0 && mantissa() == 0)
-            ? 0
-            : (isNegative()
-                ? -1
-                : 1);
+        if (exponent() == 0 && mantissa() == 0)
+            return 0;
+        return isNegative() ? -1 : 1;
     }
 
     uint16_t exponent() const

diff --git a/base/base/EnumReflection.h b/base/base/EnumReflection.h
@@ -11,7 +11,7 @@ namespace detail
 template <is_enum E, class F, size_t ...I>
 constexpr void static_for(F && f, std::index_sequence<I...>)
 {
-    (std::forward<F>(f)(std::integral_constant<E, magic_enum::enum_value<E>(I)>()) , ...);
+    (f(std::integral_constant<E, magic_enum::enum_value<E>(I)>()) , ...);
 }
 }
 

diff --git a/base/base/JSON.cpp b/base/base/JSON.cpp
@@ -651,7 +651,9 @@ std::string_view JSON::getRawString() const
     Pos s = ptr_begin;
     if (*s != '"')
         throw JSONException(std::string("JSON: expected \", got ") + *s);
-    while (++s != ptr_end && *s != '"');
+    ++s;
+    while (s != ptr_end && *s != '"')
+        ++s;
     if (s != ptr_end)
         return std::string_view(ptr_begin + 1, s - ptr_begin - 1);
     throw JSONException("JSON: incorrect syntax (expected end of string, found end of JSON).");

diff --git a/base/base/JSON.h b/base/base/JSON.h
@@ -74,7 +74,7 @@ class JSON
     const char * data() const { return ptr_begin; }
     const char * dataEnd() const { return ptr_end; }
 
-    enum ElementType
+    enum ElementType : uint8_t
     {
         TYPE_OBJECT,
         TYPE_ARRAY,

diff --git a/base/base/TypeList.h b/base/base/TypeList.h
@@ -27,7 +27,7 @@ namespace TypeListUtils /// In some contexts it's more handy to use functions in
     constexpr Root<Args...> changeRoot(TypeList<Args...>) { return {}; }
 
     template <typename F, typename ...Args>
-    constexpr void forEach(TypeList<Args...>, F && f) { (std::forward<F>(f)(TypeList<Args>{}), ...); }
+    constexpr void forEach(TypeList<Args...>, F && f) { (f(TypeList<Args>{}), ...); }
 }
 
 template <typename TypeListLeft, typename TypeListRight>

diff --git a/base/base/constexpr_helpers.h b/base/base/constexpr_helpers.h
@@ -21,7 +21,7 @@ bool func_wrapper(Func && func, Arg && arg)
 template <typename T, T Begin, typename Func, T... Is>
 constexpr bool static_for_impl(Func && f, std::integer_sequence<T, Is...>)
 {
-    return (func_wrapper(std::forward<Func>(f), std::integral_constant<T, Begin + Is>{}) || ...);
+    return (func_wrapper(f, std::integral_constant<T, Begin + Is>{}) || ...);
 }
 
 template <auto Begin, decltype(Begin) End, typename Func>

diff --git a/base/base/find_symbols.h b/base/base/find_symbols.h
@@ -147,7 +147,7 @@ constexpr uint16_t maybe_negate(uint16_t x)
         return ~x;
 }
 
-enum class ReturnMode
+enum class ReturnMode : uint8_t
 {
     End,
     Nullptr,

diff --git a/base/base/getMemoryAmount.cpp b/base/base/getMemoryAmount.cpp
@@ -77,8 +77,7 @@ uint64_t getMemoryAmountOrZero()
         {
             uint64_t limit_v1;
             if (limit_file_v1 >> limit_v1)
-                if (limit_v1 < memory_amount)
-                    memory_amount = limit_v1;
+                memory_amount = std::min(memory_amount, limit_v1);
         }
     }
 

diff --git a/base/base/hex.h b/base/base/hex.h
@@ -146,7 +146,7 @@ namespace impl
             TUInt res;
             if constexpr (sizeof(TUInt) == 1)
             {
-                res = static_cast<UInt8>(unhexDigit(data[0])) * 0x10 + static_cast<UInt8>(unhexDigit(data[1]));
+                res = unhexDigit(data[0]) * 0x10 + unhexDigit(data[1]);
             }
             else if constexpr (sizeof(TUInt) == 2)
             {
@@ -176,17 +176,19 @@ namespace impl
     };
 
     /// Helper template class to convert a value of any supported type to hexadecimal representation and back.
-    template <typename T, typename SFINAE = void>
+    template <typename T>
     struct HexConversion;
 
     template <typename TUInt>
-    struct HexConversion<TUInt, std::enable_if_t<std::is_integral_v<TUInt>>> : public HexConversionUInt<TUInt> {};
+    requires(std::is_integral_v<TUInt>)
+    struct HexConversion<TUInt> : public HexConversionUInt<TUInt> {};
 
     template <size_t Bits, typename Signed>
     struct HexConversion<wide::integer<Bits, Signed>> : public HexConversionUInt<wide::integer<Bits, Signed>> {};
 
     template <typename CityHashUInt128> /// Partial specialization here allows not to include <city.h> in this header.
-    struct HexConversion<CityHashUInt128, std::enable_if_t<std::is_same_v<CityHashUInt128, typename CityHash_v1_0_2::uint128>>>
+    requires(std::is_same_v<CityHashUInt128, typename CityHash_v1_0_2::uint128>)
+    struct HexConversion<CityHashUInt128>
     {
         static const constexpr size_t num_hex_digits = 32;