Merge branch 'apache:master' into master

.asf.yaml

@@ -52,9 +52,8 @@ github:
           - Clang Formatter
           - CheckStyle
           - P0 Regression (Doris Regression)
-          - P1 Regression (Doris Regression)
           - External Regression (Doris External Regression)
-          - cloud_p1 (Doris Cloud Regression)
+          - cloud_p0 (Doris Cloud Regression)
           - FE UT (Doris FE UT)
           - BE UT (Doris BE UT)
           - Build Broker
@@ -69,6 +68,7 @@ github:
 
       required_pull_request_reviews:
         dismiss_stale_reviews: true
+        require_code_owner_reviews: true
         required_approving_review_count: 1
     branch-1.1-lts:
       required_status_checks:
@@ -109,10 +109,8 @@ github:
         strict: false
         contexts:
           - License Check
-          - Clang Formatter
           - CheckStyle
           - P0 Regression (Doris Regression)
-          - P1 Regression (Doris Regression)
           - External Regression (Doris External Regression)
           - FE UT (Doris FE UT)
           - BE UT (Doris BE UT)

.github/CODEOWNERS

@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+be/src/io/* @platoneko @gavinchou @dataroaring
+be/src/agent/be_exec_version_manager.cpp @BiteTheDDDDt
+fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @dataroaring @CalvinKirs @morningman
+**/pom.xml @CalvinKirs @morningman
+fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java @dataroaring @morningman @yiguolei @xiaokang

.github/workflows/clang-format.yml

@@ -64,9 +64,14 @@ jobs:
           git clone https://github.com/DoozyX/clang-format-lint-action .github/actions/clang-format-lint-action
 
           pushd .github/actions/clang-format-lint-action &>/dev/null
-          git checkout 6adbe14579e5b8e19eb3e31e5ff2479f3bd302c7
+          git checkout c71d0bf4e21876ebec3e5647491186f8797fde31 # v0.18.2
           popd &>/dev/null
 
+      - name: Install Python dependencies
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'  # Adjust if needed
+
       - name: "Format it!"
         if: ${{ steps.filter.outputs.changes == 'true' }}
         uses: ./.github/actions/clang-format-lint-action

.github/workflows/code-checks.yml

@@ -50,7 +50,7 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
           sh_checker_comment: true
-          sh_checker_exclude: .git .github ^docker ^thirdparty/src ^thirdparty/installed ^ui ^docs/node_modules ^tools/clickbench-tools ^extension ^output ^fs_brokers/apache_hdfs_broker/output (^|.*/)Dockerfile$ ^be/src/apache-orc ^be/src/clucene ^pytest
+          sh_checker_exclude: .git .github ^docker ^thirdparty/src ^thirdparty/installed ^ui ^docs/node_modules ^tools/clickbench-tools ^extension ^output ^fs_brokers/apache_hdfs_broker/output (^|.*/)Dockerfile$ ^be/src/apache-orc ^be/src/clucene ^pytest ^samples
 
   preparation:
     name: "Clang Tidy Preparation"

.github/workflows/comment-to-trigger-teamcity.yml

@@ -21,22 +21,31 @@ on:
   issue_comment:
     types: [created, edited]
 
+permissions:
+  contents: read
+  pull-requests: write
+  statuses: write
+
 jobs:
   check-comment-if-need-to-trigger-teamcity:
 
     # This job only runs for pull request comments, and comment body contains 'run'
-    if: ${{ github.event.issue.pull_request && contains(github.event.comment.body, 'run') }}
+    if: ${{ github.event.issue.pull_request && (contains(github.event.comment.body, 'run') || contains(github.event.comment.body, 'skip buildall')) }}
 
     runs-on: ubuntu-latest
     env:
       COMMENT_BODY: ${{ github.event.comment.body }}
+      COMMENT_USER_ID: ${{ github.event.comment.user.id }}
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
     steps:
     - name: "Parse PR comment"
       id: parse
       run: |
         COMMENT_BODY=$(echo "${COMMENT_BODY}" | xargs)
+        PULL_REQUEST_NUM="$(echo "${{ github.event.issue.pull_request.url }}" | awk -F/ '{print $NF}')"
+        COMMIT_ID_FROM_TRIGGER="$(curl -s -H "Authorization:Bearer ${{ secrets.GITHUB_TOKEN }}" "https://api.github.com/repos/${{ github.repository }}/pulls/${PULL_REQUEST_NUM}" | jq -r '.head.sha')"
+        TARGET_BRANCH="$(curl -s -H "Authorization:Bearer ${{ secrets.GITHUB_TOKEN }}" "https://api.github.com/repos/${{ github.repository }}/pulls/${PULL_REQUEST_NUM}" | jq -r '.base.ref')"
         if [[ "${COMMENT_BODY}" == *'run buildall'* ||
             "${COMMENT_BODY}" == *'run compile'* ||
             "${COMMENT_BODY}" == *'run beut'* ||
@@ -50,15 +59,28 @@ jobs:
             "${COMMENT_BODY}" == *'run arm'* ||
             "${COMMENT_BODY}" == *'run performance'* ]]; then
             echo "comment_trigger=true" | tee -a "$GITHUB_OUTPUT"
+            echo "comment_skip=false" | tee -a "$GITHUB_OUTPUT"
+        elif [[ "${COMMENT_BODY}" == *'skip buildall'* ]]; then
+            if [[ "${COMMENT_USER_ID}" == '27881198' ||
+                  "${COMMENT_USER_ID}" == '37901441' ]]; then
+                echo "comment_trigger=false" | tee -a "$GITHUB_OUTPUT"
+                echo "comment_skip=true" | tee -a "$GITHUB_OUTPUT"
+                echo "COMMENT_USER_ID ${COMMENT_USER_ID} is allowed to skip buildall."
+            elif [[ "${COMMENT_USER_ID}" == '9208457' && "${TARGET_BRANCH}" == *'branch-2.1'* ]]; then
+                echo "COMMENT_USER_ID ${COMMENT_USER_ID} is allowed to skip buildall for branch-2.1"
+                echo "comment_trigger=false" | tee -a "$GITHUB_OUTPUT"
+                echo "comment_skip=true" | tee -a "$GITHUB_OUTPUT"
+            else
+                echo "COMMENT_USER_ID ${COMMENT_USER_ID} is not allowed to skip buildall."
+                exit
+            fi
         else
             echo "comment_trigger=false" | tee -a "$GITHUB_OUTPUT"
+            echo "comment_skip=false" | tee -a "$GITHUB_OUTPUT"
             echo "find no keyword in comment body, skip this action."
             exit
         fi
 
-        PULL_REQUEST_NUM="$(echo "${{ github.event.issue.pull_request.url }}" | awk -F/ '{print $NF}')"
-        COMMIT_ID_FROM_TRIGGER="$(curl -s -H "Authorization:Bearer ${{ secrets.GITHUB_TOKEN }}" "https://api.github.com/repos/${{ github.repository }}/pulls/${PULL_REQUEST_NUM}" | jq -r '.head.sha')"
-        TARGET_BRANCH="$(curl -s -H "Authorization:Bearer ${{ secrets.GITHUB_TOKEN }}" "https://api.github.com/repos/${{ github.repository }}/pulls/${PULL_REQUEST_NUM}" | jq -r '.base.ref')"
         echo "PULL_REQUEST_NUM=${PULL_REQUEST_NUM}" | tee -a "$GITHUB_OUTPUT"
         echo "COMMIT_ID_FROM_TRIGGER=${COMMIT_ID_FROM_TRIGGER}" | tee -a "$GITHUB_OUTPUT"
         echo "TARGET_BRANCH='${TARGET_BRANCH}'" | tee -a "$GITHUB_OUTPUT"
@@ -71,7 +93,7 @@ jobs:
         echo "COMMENT_REPEAT_TIMES=${COMMENT_REPEAT_TIMES}" | tee -a "$GITHUB_OUTPUT"
 
     - name: "Checkout master"
-      if: ${{ fromJSON(steps.parse.outputs.comment_trigger) }}
+      if: ${{ fromJSON(steps.parse.outputs.comment_trigger) || fromJSON(steps.parse.outputs.comment_skip) }}
       uses: actions/checkout@v4
 
     - name: "Check if pr need run build"
@@ -150,6 +172,7 @@ jobs:
         set -x
         if [[ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'master'" ||
               "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'branch-2.0'" ||
+              "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'branch-3.0'"  ||
               "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'branch-2.1'" ]]; then
           trigger_or_skip_build \
             "${{ steps.changes.outputs.changed_fe_ut }}" \
@@ -158,7 +181,7 @@ jobs:
             "feut" \
             "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
         else
-          echo "PR target branch is not in (master, branch-2.0, branch-2.1), skip run feut"
+          echo "PR target branch is not in (master, branch-2.0, branch-2.1, branch-3.0), skip run feut"
           trigger_or_skip_build \
             "false" \
             "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
@@ -175,6 +198,7 @@ jobs:
         set -x
         if [[ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'master'" ||
               "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'branch-2.0'" ||
+              "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'branch-3.0'" ||
               "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'branch-2.1'" ]]; then
           trigger_or_skip_build \
             "${{ steps.changes.outputs.changed_be_ut }}" \
@@ -183,7 +207,7 @@ jobs:
             "beut" \
             "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
         else
-          echo "PR target branch is not in (master, branch-2.0, branch-2.1), skip run beut"
+          echo "PR target branch is not in (master, branch-2.0, branch-2.1, branch-3.0), skip run beut"
           trigger_or_skip_build \
             "false" \
             "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
@@ -197,15 +221,16 @@ jobs:
       run: |
         source ./regression-test/pipeline/common/teamcity-utils.sh
         set -x
-        if [[ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'master'" ]]; then
+        if [[ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'master'" ||
+              "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'branch-3.0'" ]]; then
           trigger_or_skip_build \
             "${{ steps.changes.outputs.changed_cloud_ut }}" \
             "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
             "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
             "cloudut" \
             "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
         else
-          echo "PR target branch is not master, skip run cloudut"
+          echo "PR target branch is not in (master, branch-3.0), skip run cloudut"
         fi
 
     - name: "Trigger or Skip compile"
@@ -279,16 +304,17 @@ jobs:
           echo "COMMENT_TRIGGER_TYPE is buildall, trigger compile is enough, compile will trigger cloud_p0" && exit
         fi
         set -x
-        if [[ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'master'" ]]; then
-          echo "PR target branch in (master), need run cloud_p0"
+        if [[ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'master'" ||
+              "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'branch-3.0'" ]]; then
+          echo "PR target branch is in (master, branch-3.0), need run cloud_p0"
           trigger_or_skip_build \
             "${{ steps.changes.outputs.changed_cloud_p0 }}" \
             "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
             "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
             "cloud_p0" \
             "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
         else
-          echo "PR target branch not in (master), skip run cloud_p0"
+          echo "PR target branch is not in (master, branch-3.0), skip run cloud_p0"
           trigger_or_skip_build \
             "false" \
             "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
@@ -305,16 +331,17 @@ jobs:
           echo "COMMENT_TRIGGER_TYPE is buildall, trigger compile is enough, compile will trigger cloud_p1" && exit
         fi
         set -x
-        if [[ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'master'" ]]; then
-          echo "PR target branch in (master), need run cloud_p1"
+        if [[ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'master'" ||
+              "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'branch-3.0'" ]]; then
+          echo "PR target branch is in (master, branch-3.0), need run cloud_p1"
           trigger_or_skip_build \
             "${{ steps.changes.outputs.changed_cloud_p1 }}" \
             "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
             "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
             "cloud_p1" \
             "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
         else
-          echo "PR target branch not in (master), skip run cloud_p1"
+          echo "PR target branch is not in (master, branch-3.0), skip run cloud_p1"
           trigger_or_skip_build \
             "false" \
             "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
@@ -341,20 +368,37 @@ jobs:
         source ./regression-test/pipeline/common/teamcity-utils.sh
         set -x
         if [[ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'master'" ||
+              "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'branch-3.0'" ||
               "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'branch-2.0'" ]]; then
-          echo "PR target branch in (master, branch-2.0), need run performance"
+          echo "PR target branch is in (master, branch-2.0, branch-3.0), need run performance"
           trigger_or_skip_build \
             "${{ steps.changes.outputs.changed_performance }}" \
             "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
             "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
             "performance" \
             "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
         else
-          echo "PR target branch not in (master, branch-2.0), skip run performance"
+          echo "PR target branch is not in (master, branch-2.0, branch-3.0), skip run performance"
           trigger_or_skip_build \
             "false" \
             "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
             "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
             "performance" \
             "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
         fi
+
+    - name: "Skip buildall"
+      if: ${{ fromJSON(steps.parse.outputs.comment_skip) }}
+      run: |
+        source ./regression-test/pipeline/common/teamcity-utils.sh
+        skip_build "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" feut
+        skip_build "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" beut
+        skip_build "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" compile
+        skip_build "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" p0
+        skip_build "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" p1
+        skip_build "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" external
+        skip_build "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" performance
+        skip_build "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" arm
+        skip_build "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" cloud_p0
+        skip_build "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" cloud_p1
+        skip_build "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" cloudut

.github/workflows/labeler/scope-label-conf.yml

@@ -23,3 +23,8 @@ meta-change:
   - fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
   - gensrc/thrift/*
   - gensrc/proto/*
+
+doing:
+  - base-branch: 'master'
+  - changed-files:
+    - any-glob-to-any-file: '**'

.github/workflows/lfs-warning.yml

@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+name: 'Check Large File'
+
+on: [push, pull_request_target]
+
+jobs:
+  large-file-checker:
+    name: "Check large file"
+    runs-on: ubuntu-latest
+    steps:
+      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+        uses: actions/checkout@v3
+        with:
+          persist-credentials: false
+          submodules: recursive
+
+      - name: "Checkout lfs-warning commit"
+        run: |
+          rm -rf ./.github/actions/lfs-warning
+          git clone https://github.com/ppremk/lfs-warning .github/actions/lfs-warning
+          pushd .github/actions/lfs-warning &>/dev/null
+          git checkout 4b98a8a5e6c429c23c34eee02d71553bca216425
+          popd &>/dev/null
+
+      - name: "Check Large File"
+        uses: ./.github/actions/lfs-warning
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          filesizelimit: 1MB
+

.github/workflows/sonarcloud.yml

@@ -41,11 +41,11 @@ jobs:
               - 'fe/**'
               - 'gensrc/proto/**'
               - 'gensrc/thrift/**'
-      - name: Set up JDK 11
+      - name: Set up JDK 17
         if: ${{ steps.filter.outputs.fe_changes == 'true' }}
         uses: actions/setup-java@v3
         with:
-          java-version: 11
+          java-version: 17
           distribution: 'adopt'
       - name: Cache SonarCloud packages
         if: ${{ steps.filter.outputs.fe_changes == 'true' }}

.gitignore

@@ -60,6 +60,8 @@ docker/thirdparties/docker-compose/hive/scripts/paimon1
 fe_plugins/output
 fe_plugins/**/.factorypath
 
+docker/thirdparties/docker-compose/hive/scripts/data/*/*/data
+
 fs_brokers/apache_hdfs_broker/src/main/resources/
 fs_brokers/apache_hdfs_broker/src/main/thrift/
 
@@ -100,7 +102,6 @@ be/tags
 be/test/olap/test_data/tablet_meta_test.hdr
 be/.devcontainer/
 be/src/apache-orc/
-zoneinfo/
 
 # Cloud
 cloud/build*/