Merge branch 'master' into master-dec-job-docs

.asf.yaml

@@ -59,8 +59,7 @@ github:
           - BE UT (Doris BE UT)
           - Build Broker
           - Build Documents
-          - ShellCheck
-          - clickbench-new (clickbench)
+          - ShellCheck          
           - Build Third Party Libraries (Linux)
           - Build Third Party Libraries (macOS)
           - COMPILE (DORIS_COMPILE)
@@ -94,9 +93,8 @@ github:
           - FE UT (Doris FE UT)
           - BE UT (Doris BE UT)
           - Build Broker
-          - Build Documents
           - ShellCheck
-          - clickbench-new (clickbench)
+          - Build Documents
           - Build Third Party Libraries (Linux)
           - Build Third Party Libraries (macOS)
           - COMPILE (DORIS_COMPILE)
@@ -109,14 +107,13 @@ github:
     - LemonLiTree
     - Yukang-Lian
     - TangSiyang2001
-    - platoneko
     - Lchangliang
     - freemandealer
-    - nanfeng1999
-    - gitccl
     - shuke987
     - wm1581066
     - KassieZ
+    - yujun777
+    - gavinchou
 
 notifications:
     pullrequests_status:  [email protected]

.clang-tidy

@@ -4,6 +4,10 @@ Checks: |
   clang-diagnostic-*,
   clang-analyzer-*,
   bugprone-redundant-branch-condition,
+  bugprone-use-after-move,
+  bugprone-bool-pointer-implicit-conversion,
+  bugprone-unused-raii,
+  bugprone-fold-init-type,
   modernize-*,
   -modernize-use-trailing-return-type,
   -modernize-use-nodiscard,
@@ -24,10 +28,9 @@ Checks: |
   performance-faster-string-find,
   performance-inefficient-algorithm,
   performance-move-const-arg
-WarningsAsErrors: '*'
+WarningsAsErrors: "*"
 CheckOptions:
-  - key:             readability-function-size.LineThreshold
-    value:           '80'
-  - key:             readability-function-cognitive-complexity.Threshold
-    value:           '50'
-
+  - key: readability-function-size.LineThreshold
+    value: "80"
+  - key: readability-function-cognitive-complexity.Threshold
+    value: "50"

.github/workflows/code-checks.yml

@@ -79,15 +79,16 @@ jobs:
         id: filter
         with:
           filters: |
-            be_changes:
+            cpp_changes:
               - 'be/**'
+              - 'cloud/**'
               - 'gensrc/proto/**'
               - 'gensrc/thrift/**'
 
       - name: Generate compile_commands.json
         id: generate
         run: |
-          if [[ "${{ steps.filter.outputs.be_changes }}" == 'true' ]]; then
+          if [[ "${{ steps.filter.outputs.cpp_changes }}" == 'true' ]]; then
             export DEFAULT_DIR='/opt/doris'
 
             mkdir "${DEFAULT_DIR}"
@@ -104,14 +105,14 @@ jobs:
             popd
 
             export PATH="${DEFAULT_DIR}/ldb-toolchain/bin/:$(pwd)/thirdparty/installed/bin/:${PATH}"
-            DISABLE_JAVA_UDF=ON DORIS_TOOLCHAIN=clang ENABLE_PCH=OFF OUTPUT_BE_BINARY=0 ./build.sh --be
+            DISABLE_JAVA_UDF=ON DORIS_TOOLCHAIN=clang ENABLE_PCH=OFF OUTPUT_BE_BINARY=0 ./build.sh --be --cloud
           fi
 
-          echo "should_check=${{ steps.filter.outputs.be_changes }}" >>${GITHUB_OUTPUT}
+          echo "should_check=${{ steps.filter.outputs.cpp_changes }}" >>${GITHUB_OUTPUT}
 
       - name: Upload
         uses: actions/upload-artifact@v3
-        if: ${{ steps.filter.outputs.be_changes == 'true' }}
+        if: ${{ steps.filter.outputs.cpp_changes == 'true' }}
         with:
           name: compile_commands
           path: ./be/build_Release/compile_commands.json

.github/workflows/comment-to-trigger-teamcity.yml

@@ -41,13 +41,14 @@ jobs:
             "${COMMENT_BODY}" == *'run compile'* ||
             "${COMMENT_BODY}" == *'run beut'* ||
             "${COMMENT_BODY}" == *'run feut'* ||
+            "${COMMENT_BODY}" == *'run cloudut'* ||
             "${COMMENT_BODY}" == *'run p0'* ||
             "${COMMENT_BODY}" == *'run p1'* ||
             "${COMMENT_BODY}" == *'run external'* ||
             "${COMMENT_BODY}" == *'run pipelinex_p0'* ||
-            "${COMMENT_BODY}" == *'run clickbench'* ||
+            "${COMMENT_BODY}" == *'run cloud_p0'* ||
             "${COMMENT_BODY}" == *'run arm'* ||
-            "${COMMENT_BODY}" == *'run tpch'* ]]; then
+            "${COMMENT_BODY}" == *'run performance'* ]]; then
             echo "comment_trigger=true" | tee -a "$GITHUB_OUTPUT"
         else
             echo "comment_trigger=false" | tee -a "$GITHUB_OUTPUT"
@@ -63,7 +64,7 @@ jobs:
         echo "TARGET_BRANCH='${TARGET_BRANCH}'" | tee -a "$GITHUB_OUTPUT"
         echo "COMMENT_BODY='${COMMENT_BODY}'" | tee -a "$GITHUB_OUTPUT"
 
-        reg="run (buildall|compile|p0|p1|feut|beut|external|clickbench|pipelinex_p0|arm|tpch)( [1-9]*[0-9]+)*"
+        reg="run (buildall|compile|p0|p1|feut|beut|cloudut|external|clickbench|pipelinex_p0|cloud_p0|arm|performance)( [1-9]*[0-9]+)*"
         COMMENT_TRIGGER_TYPE="$(echo -e "${COMMENT_BODY}" | xargs | grep -E "${reg}" | awk -F' ' '{print $2}' | sed -n 1p | sed 's/\r//g')"
         COMMENT_REPEAT_TIMES="$(echo -e "${COMMENT_BODY}" | xargs | grep -E "${reg}" | awk -F' ' '{print $3}' | sed -n 1p | sed 's/\r//g')"
         echo "COMMENT_TRIGGER_TYPE=${COMMENT_TRIGGER_TYPE}" | tee -a "$GITHUB_OUTPUT"
@@ -90,12 +91,10 @@ jobs:
           else
             echo "changed_be_ut=false" | tee -a "$GITHUB_OUTPUT"
           fi
-          if file_changed_ckb; then
-            echo "changed_ckb=true" | tee -a "$GITHUB_OUTPUT"
-            echo "changed_tpch=true" | tee -a "$GITHUB_OUTPUT"
+          if file_changed_cloud_ut; then
+            echo "changed_cloud_ut=true" | tee -a "$GITHUB_OUTPUT"
           else
-            echo "changed_ckb=false" | tee -a "$GITHUB_OUTPUT"
-            echo "changed_tpch=false" | tee -a "$GITHUB_OUTPUT"
+            echo "changed_cloud_ut=false" | tee -a "$GITHUB_OUTPUT"
           fi
           if file_changed_regression_p0; then
             echo "changed_p0=true" | tee -a "$GITHUB_OUTPUT"
@@ -113,17 +112,28 @@ jobs:
           else
             echo "changed_p1=false" | tee -a "$GITHUB_OUTPUT"
           fi
+          if file_changed_performance; then
+            echo "changed_performance=true" | tee -a "$GITHUB_OUTPUT"
+          else
+            echo "changed_performance=false" | tee -a "$GITHUB_OUTPUT"
+          fi
+          if file_changed_cloud_p0; then
+            echo "changed_cloud_p0=true" | tee -a "$GITHUB_OUTPUT"
+          else
+            echo "changed_cloud_p0=false" | tee -a "$GITHUB_OUTPUT"
+          fi
         else
           echo "INFO: failed to _get_pr_changed_files, default trigger all"
           echo "changed_fe_ut=true" | tee -a "$GITHUB_OUTPUT"
           echo "changed_be_ut=true" | tee -a "$GITHUB_OUTPUT"
-          echo "changed_ckb=true" | tee -a "$GITHUB_OUTPUT"
-          echo "changed_tpch=true" | tee -a "$GITHUB_OUTPUT"
+          echo "changed_cloud_ut=true" | tee -a "$GITHUB_OUTPUT"
           echo "changed_p0=true" | tee -a "$GITHUB_OUTPUT"
           echo "changed_external=true" | tee -a "$GITHUB_OUTPUT"
           echo "changed_pipelinex_p0=true" | tee -a "$GITHUB_OUTPUT"
           echo "changed_arm=true" | tee -a "$GITHUB_OUTPUT"
           echo "changed_p1=true" | tee -a "$GITHUB_OUTPUT"
+          echo "changed_performance=true" | tee -a "$GITHUB_OUTPUT"
+          echo "changed_cloud_p0=true" | tee -a "$GITHUB_OUTPUT"
         fi
 
     # - name: "Setup tmate session"
@@ -155,6 +165,18 @@ jobs:
           "beut" \
           "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
 
+    - name: "Trigger or Skip cloudut"
+      if: ${{ fromJSON(steps.parse.outputs.comment_trigger) && contains(fromJSON('["cloudut", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }}
+      run: |
+        source ./regression-test/pipeline/common/teamcity-utils.sh
+        set -x
+        trigger_or_skip_build \
+          "${{ steps.changes.outputs.changed_cloud_ut }}" \
+          "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
+          "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
+          "cloudut" \
+          "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
+
     - name: "Trigger or Skip compile"
       if: ${{ fromJSON(steps.parse.outputs.comment_trigger) && contains(fromJSON('["compile", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }}
       run: |
@@ -228,41 +250,53 @@ jobs:
           "pipelinex_p0" \
           "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
 
-    - name: "Trigger or Skip arm"
-      if: ${{ fromJSON(steps.parse.outputs.comment_trigger) && contains(fromJSON('["arm", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }}
+    - name: "Trigger or Skip cloud_p0"
+      if: ${{ fromJSON(steps.parse.outputs.comment_trigger) && contains(fromJSON('["cloud_p0", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }}
       run: |
         source ./regression-test/pipeline/common/teamcity-utils.sh
+        if [[ ${{ steps.parse.outputs.COMMENT_TRIGGER_TYPE }} == "buildall" ]]; then
+          echo "COMMENT_TRIGGER_TYPE is buildall, trigger compile is enough, compile will trigger cloud_p0" && exit
+        fi
         set -x
         trigger_or_skip_build \
-          "${{ steps.changes.outputs.changed_arm }}" \
+          "${{ steps.changes.outputs.changed_cloud_p0 }}" \
           "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
           "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
-          "arm" \
+          "cloud_p0" \
           "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
 
-    - name: "Trigger or Skip clickbench"
-      if: ${{ fromJSON(steps.parse.outputs.comment_trigger) && contains(fromJSON('["clickbench", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }}
+    - name: "Trigger or Skip arm"
+      if: ${{ fromJSON(steps.parse.outputs.comment_trigger) && contains(fromJSON('["arm", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }}
       run: |
         source ./regression-test/pipeline/common/teamcity-utils.sh
         set -x
         trigger_or_skip_build \
-          "${{ steps.changes.outputs.changed_ckb }}" \
+          "${{ steps.changes.outputs.changed_arm }}" \
           "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
           "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
-          "clickbench" \
+          "arm" \
           "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
 
-    - name: "Trigger or Skip tpch"
-      if: ${{ fromJSON(steps.parse.outputs.comment_trigger) && contains(fromJSON('["tpch", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }}
+    - name: "Trigger or Skip performance"
+      if: ${{ fromJSON(steps.parse.outputs.comment_trigger) && contains(fromJSON('["performance", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }}
       run: |
         source ./regression-test/pipeline/common/teamcity-utils.sh
-        if [[ ${{ steps.parse.outputs.COMMENT_TRIGGER_TYPE }} == "buildall" ]]; then
-          echo "COMMENT_TRIGGER_TYPE is buildall, trigger clickbench is enough, clickbench will trigger tpch" && exit
-        fi
         set -x
-        trigger_or_skip_build \
-          "${{ steps.changes.outputs.changed_tpch }}" \
-          "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
-          "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
-          "tpch" \
-          "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
+        if [[ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'master'" ||
+              "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'branch-2.0'" ]]; then
+          echo "PR target branch in (master, branch-2.0), need run performance"
+          trigger_or_skip_build \
+            "${{ steps.changes.outputs.changed_performance }}" \
+            "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
+            "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
+            "performance" \
+            "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
+        else
+          echo "PR target branch not in (master, branch-2.0), skip run performance"
+          trigger_or_skip_build \
+            "false" \
+            "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
+            "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
+            "performance" \
+            "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
+        fi

.gitignore

@@ -54,6 +54,8 @@ thirdparty/installed*
 thirdparty/doris-thirdparty*.tar.xz
 
 docker/thirdparties/docker-compose/mysql/data
+docker/thirdparties/docker-compose/hive/scripts/tpch1.db/
+docker/thirdparties/docker-compose/hive/scripts/paimon1
 
 fe_plugins/output
 fe_plugins/**/.factorypath
@@ -99,6 +101,11 @@ be/.devcontainer/
 be/src/apache-orc/
 zoneinfo/
 
+# Cloud
+cloud/build*/
+cloud/cmake-build*/
+cloud/ut_build*/
+
 ## tools
 tools/ssb-tools/ssb-data/
 tools/ssb-tools/ssb-dbgen/

.gitmodules

@@ -24,6 +24,3 @@
 	path = be/src/clucene
 	url = https://github.com/apache/doris-thirdparty.git
 	branch = clucene
-[submodule ".github/actions/create-or-update-comment"]
-	path = .github/actions/create-or-update-comment
-	url = https://github.com/peter-evans/create-or-update-comment.git

.licenserc.yaml

@@ -88,6 +88,7 @@ header:
     - "conf/mysql_ssl_default_certificate/client_certificate/client-cert.pem"
     - "conf/mysql_ssl_default_certificate/client_certificate/client-key.pem"
     - "regression-test/ssl_default_certificate/*"
+    - "regression-test/pipeline/**"
     - "extension/beats/go.mod"
     - "extension/beats/go.sum"
     - "pytest/hdfs"