Merge branch 'apache:master' into master

apache · Jan 26, 2024 · b9ff770 · b9ff770
2 parents 6827a65 + 47a1091
commit b9ff770
Show file tree

Hide file tree

Showing 217 changed files with 5,859 additions and 2,867 deletions.
diff --git a/.github/PULL_REQUEST_TEMPLATE b/.github/PULL_REQUEST_TEMPLATE
@@ -29,28 +29,10 @@ Please include a summary of the change and which issue is fixed. Please also inc
 
 ---
 
-# Checklists
-## 📝 Author Self Checklist
+# Checklist 📝
 <!--- Go over all the following points, and put an `x` in all the boxes that apply. -->
 <!--- If you're unsure about any of these, don't hesitate to ask. We're here to help! -->
-- [ ] My code follows the [style guidelines](https://kyuubi.readthedocs.io/en/master/contributing/code/style.html) of this project
-- [ ] I have performed a self-review
-- [ ] I have commented my code, particularly in hard-to-understand areas
-- [ ] I have made corresponding changes to the documentation
-- [ ] My changes generate no new warnings
-- [ ] I have added tests that prove my fix is effective or that my feature works
-- [ ] New and existing unit tests pass locally with my changes
-- [ ] This patch was not authored or co-authored using [Generative Tooling](https://www.apache.org/legal/generative-tooling.html)
-
-## 📝 Committer Pre-Merge Checklist
-
-- [ ] Pull request title is okay.
-- [ ] No license issues.
-- [ ] Milestone correctly set?
-- [ ] Test coverage is ok
-- [ ] Assignees are selected.
-- [ ] Minimum number of approvals
-- [ ] No changes are requested
 
+- [ ] This patch was not authored or co-authored using [Generative Tooling](https://www.apache.org/legal/generative-tooling.html)
 
 **Be nice. Be informative.**
diff --git a/.github/labeler.yml b/.github/labeler.yml
@@ -19,109 +19,179 @@
 # Pull Request Labeler Github Action Configuration: https://github.com/marketplace/actions/labeler
 
 "kind:build":
-  - ".dockerignore"
-  - ".rat-excludes"
-  - ".scalafmt"
-  - "**/*pom.xml"
-  - "bin/docker-image-tool.sh"
-  - "build/**/*"
-  - "docker/**/*"
-  - "docs/requirements"
-  - "kyuubi-assembly/**/*"
-  - "scalastyle-config.xml"
-  - any: ["dev/**/*", "!dev/kyuubi-codecov/**/*", "!dev/kyuubi-tpcds/**/*"]
+  - changed-files:
+    - any-glob-to-any-file: [
+      '.dockerignore',
+      '.rat-excludes',
+      '.scalafmt',
+      '**/*pom.xml',
+      'bin/docker-image-tool.sh',
+      'build/**/*',
+      'docker/**/*',
+      'docs/requirements',
+      'kyuubi-assembly/**/*',
+      'scalastyle-config.xml'
+    ]
+    - all-globs-to-any-file: [
+      'dev/**/*',
+      '!dev/kyuubi-codecov/**/*',
+      '!dev/kyuubi-tpcds/**/*'
+    ]
 
 "kind:deploy":
-  - any: ["bin/**/*", "!bin/beeline", "!bin/docker-image-tool.sh"]
+  - changed-files:
+    - all-globs-to-any-file: [
+      'bin/**/*',
+      '!bin/beeline',
+      '!bin/docker-image-tool.sh'
+    ]
 
 "kind:documentation":
-  - "*.md"
-  - "conf/**/*"
-  - "docs/**/*"
-  - "readthedocs.yml"
+  - changed-files:
+    - any-glob-to-any-file: [
+      '*.md',
+      'conf/**/*',
+      'docs/**/*',
+      'readthedocs.yml'
+    ]
 
 "kind:infra":
-  - ".asf.yaml"
-  - ".gitattributes"
-  - ".github/**/*"
-  - ".gitignore"
-  - "LICENSE"
-  - "LICENSE-binary"
-  - "NOTICE"
-  - "NOTICE-binary"
-  - "codecov.yml"
-  - "dev/kyuubi-codecov/**/*"
-  - "licenses-binary"
+  - changed-files:
+    - any-glob-to-any-file: [
+      '.asf.yaml',
+      '.gitattributes',
+      '.github/**/*',
+      '.gitignore',
+      'LICENSE',
+      'LICENSE-binary',
+      'NOTICE',
+      'NOTICE-binary',
+      'codecov.yml',
+      'dev/kyuubi-codecov/**/*',
+      'licenses-binary'
+    ]
 
 "module:common":
-  - "kyuubi-common/**/*"
+  - changed-files:
+    - any-glob-to-any-file: [
+      'kyuubi-common/**/*'
+    ]
 
 "module:ctl":
-  - "bin/beeline"
-  - "kyuubi-ctl/**/*"
-  - "kyuubi-hive-beeline/**/*"
-  - "kyuubi-hive-jdbc/**/*"
-  - "kyuubi-hive-jdbc-shaded/**/*"
+  - changed-files:
+    - any-glob-to-any-file: [
+      'bin/beeline',
+      'kyuubi-ctl/**/*',
+      'kyuubi-hive-beeline/**/*',
+      'kyuubi-hive-jdbc/**/*',
+      'kyuubi-hive-jdbc-shaded/**/*'
+    ]
 
 "module:events":
-  - "kyuubi-events/**/*"
+  - changed-files:
+    - any-glob-to-any-file: [
+      'kyuubi-events/**/*'
+    ]
 
 "module:flink":
-  - "externals/kyuubi-flink-sql-engine/**/*"
-  - "integration-tests/kyuubi-flink-it/**/*"
+  - changed-files:
+    - any-glob-to-any-file: [
+      'externals/kyuubi-flink-sql-engine/**/*',
+      'integration-tests/kyuubi-flink-it/**/*'
+    ]
 
 "module:ha":
-  - "kyuubi-ha/**/*"
-  - "kyuubi-zookeeper/**/*"
+  - changed-files:
+    - any-glob-to-any-file: [
+      'kyuubi-ha/**/*',
+      'kyuubi-zookeeper/**/*'
+    ]
 
 "module:hive":
-  - "bin/beeline"
-  - "externals/kyuubi-hive-sql-engine/**/*"
-  - "kyuubi-hive-beeline/**/*"
-  - "kyuubi-hive-jdbc/**/*"
-  - "kyuubi-hive-jdbc-shaded/**/*"
+  - changed-files:
+    - any-glob-to-any-file: [
+      'bin/beeline',
+      'externals/kyuubi-hive-sql-engine/**/*',
+      'kyuubi-hive-beeline/**/*',
+      'kyuubi-hive-jdbc/**/*',
+      'kyuubi-hive-jdbc-shaded/**/*'
+    ]
 
 "module:jdbc":
-  - "externals/kyuubi-jdbc-engine/**/*"
+  - changed-files:
+    - any-glob-to-any-file: [
+      'externals/kyuubi-jdbc-engine/**/*'
+    ]
 
 "module:kubernetes":
-  - ".dockerignore"
-  - "bin/docker-image-tool.sh"
-  - "docker/**/*"
-  - "integration-tests/kyuubi-kubernetes-it/**/*"
-  - "tools/spark-block-cleaner/**/*"
+  - changed-files:
+    - any-glob-to-any-file: [
+      '.dockerignore',
+      'bin/docker-image-tool.sh',
+      'docker/**/*',
+      'integration-tests/kyuubi-kubernetes-it/**/*'
+    ]
 
 "module:metrics":
-  - "kyuubi-metrics/**/*"
+  - changed-files:
+    - any-glob-to-any-file: [
+      'kyuubi-metrics/**/*'
+    ]
 
 "module:trino":
-  - "externals/kyuubi-trino-engine/**/*" 
+  - changed-files:
+    - any-glob-to-any-file: [
+      'externals/kyuubi-trino-engine/**/*'
+    ]
 
 "module:tpcds":
-  - "dev/kyuubi-tpcds/**/*"
+  - changed-files:
+    - any-glob-to-any-file: [
+      'dev/kyuubi-tpcds/**/*'
+    ]
 
 "module:server":
-  - "bin/kyuubi"
-  - "kyuubi-server/src/**/*"
-  - "kyuubi-server/pom.xml"
-  - "extension/server/kyuubi-server-plugin/**/*"
+  - changed-files:
+    - any-glob-to-any-file: [
+      'bin/kyuubi',
+      'kyuubi-server/src/**/*',
+      'kyuubi-server/pom.xml',
+      'extension/server/kyuubi-server-plugin/**/*'
+    ]
 
 "module:spark":
-  - "externals/kyuubi-spark-sql-engine/**/*" 
-  - "extensions/spark/**/*"
-  - "tools/spark-block-cleaner/**/*"
+  - changed-files:
+      - any-glob-to-any-file: [
+        'externals/kyuubi-spark-sql-engine/**/*',
+        'extensions/spark/**/*'
+      ]
 
 "module:extensions":
-  - "extensions/**/*"
+  - changed-files:
+    - any-glob-to-any-file: [
+      'extensions/**/*'
+    ]
 
 "module:rest-client":
-  - "kyuubi-rest-client/**/*"
+  - changed-files:
+    - any-glob-to-any-file: [
+      'kyuubi-rest-client/**/*'
+    ]
 
 "module:integration-tests":
-  - "integration-tests/**/*"
+  - changed-files:
+    - any-glob-to-any-file: [
+      'integration-tests/**/*'
+    ]
 
 "module:authz":
-  - "extensions/spark/kyuubi-spark-authz/**/*"
+  - changed-files:
+    - any-glob-to-any-file: [
+      'extensions/spark/kyuubi-spark-authz/**/*'
+    ]
 
 "module:ui":
-  - "kyuubi-server/web-ui/**/*"
+  - changed-files:
+    - any-glob-to-any-file: [
+      'kyuubi-server/web-ui/**/*'
+    ]
diff --git a/.github/workflows/gluten.yml b/.github/workflows/gluten.yml
@@ -99,11 +99,6 @@ jobs:
         with:
           path: gluten/package/target/
           key: gluten_package_${{ steps.date.outputs.date }}
-      - name: Cache Gluten Package
-        uses: actions/cache@v3
-        with:
-          path: gluten/package/target/
-          key: gluten_package
       - name: Setup JDK 8
         uses: actions/setup-java@v4
         with:

diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml
@@ -28,7 +28,7 @@ jobs:
   triage:
     runs-on: ubuntu-22.04
     steps:
-    - uses: actions/labeler@v4
+    - uses: actions/labeler@v5
       with:
         repo-token: "${{ secrets.GITHUB_TOKEN }}"
         sync-labels: true
diff --git a/.github/workflows/license.yml b/.github/workflows/license.yml
@@ -44,7 +44,7 @@ jobs:
           check-latest: false
       - run: >-
           build/mvn org.apache.rat:apache-rat-plugin:check
-          -Ptpcds -Pspark-block-cleaner -Pkubernetes-it
+          -Ptpcds -Pkubernetes-it
           -Pspark-3.1 -Pspark-3.2 -Pspark-3.3 -Pspark-3.4 -Pspark-3.5
       - name: Upload rat report
         if: failure()

diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
@@ -356,9 +356,9 @@ jobs:
         uses: ./.github/actions/cache-engine-archives
       - name: Build and test Trino with maven w/o linters
         run: |
-          TEST_MODULES="kyuubi-server,externals/kyuubi-trino-engine,externals/kyuubi-spark-sql-engine,externals/kyuubi-download,integration-tests/kyuubi-trino-it"
-          ./build/mvn ${MVN_OPT} -pl ${TEST_MODULES} -am -Pflink-provided -Phive-provided clean install -DskipTests
-          ./build/mvn -Dmaven.javadoc.skip=true -Drat.skip=true -Dscalastyle.skip=true -Dspotless.check.skip -pl ${TEST_MODULES} -am -Pflink-provided -Phive-provided test -Dtest=none -DwildcardSuites=org.apache.kyuubi.it.trino.operation.TrinoOperationSuite,org.apache.kyuubi.it.trino.server.TrinoFrontendSuite
+          TEST_MODULES="externals/kyuubi-trino-engine,integration-tests/kyuubi-trino-it"
+          ./build/mvn ${MVN_OPT} -pl ${TEST_MODULES} -am clean install -DskipTests
+          ./build/mvn ${MVN_OPT} -pl ${TEST_MODULES} test
       - name: Upload test logs
         if: failure()
         uses: actions/upload-artifact@v3
@@ -410,7 +410,7 @@ jobs:
         with:
           # passthrough CI into build container
           build-args: |
-            CI=${CI} 
+            CI=${CI}
             MVN_ARG=--flink-provided --hive-provided -Dmaven.javadoc.skip=true -Drat.skip=true -Dscalastyle.skip=true -Dspotless.check.skip -DskipTests
           context: .
           file: build/Dockerfile
@@ -427,8 +427,8 @@ jobs:
           # https://minikube.sigs.k8s.io/docs/handbook/pushing/#7-loading-directly-to-in-cluster-container-runtime
           minikube image load apache/kyuubi:latest
           # pre-install spark into minikube
-          docker pull apache/spark:3.4.1
-          minikube image load apache/spark:3.4.1
+          docker pull apache/spark:3.4.2
+          minikube image load apache/spark:3.4.2
       - name: kubectl pre-check
         run: |
           kubectl get nodes

diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml
@@ -34,7 +34,7 @@ jobs:
     strategy:
       matrix:
         profiles:
-          - '-Pflink-provided,hive-provided,spark-provided,spark-block-cleaner,spark-3.5,spark-3.4,spark-3.3,spark-3.2,tpcds,kubernetes-it'
+          - '-Pflink-provided,hive-provided,spark-provided,spark-3.5,spark-3.4,spark-3.3,spark-3.2,tpcds,kubernetes-it'
 
     steps:
       - uses: actions/checkout@v4
@@ -65,7 +65,7 @@ jobs:
         if: steps.modules-check.conclusion == 'success' && steps.modules-check.outcome == 'failure'
         run: |
           MVN_OPT="-DskipTests -Dorg.slf4j.simpleLogger.defaultLogLevel=warn -Dmaven.javadoc.skip=true -Drat.skip=true -Dscalastyle.skip=true -Dspotless.check.skip"
-          build/mvn clean install ${MVN_OPT} -Pflink-provided,hive-provided,spark-provided,spark-block-cleaner,spark-3.2,tpcds
+          build/mvn clean install ${MVN_OPT} -Pflink-provided,hive-provided,spark-provided,spark-3.2,tpcds
           build/mvn clean install ${MVN_OPT} -pl extensions/spark/kyuubi-extension-spark-3-1 -Pspark-3.1
           build/mvn clean install ${MVN_OPT} -pl extensions/spark/kyuubi-extension-spark-3-3,extensions/spark/kyuubi-spark-connector-hive -Pspark-3.3
           build/mvn clean install ${MVN_OPT} -pl extensions/spark/kyuubi-extension-spark-3-4 -Pspark-3.4

diff --git a/.gitignore b/.gitignore
@@ -80,3 +80,7 @@ conf/kyuubi-env.sh
 kyuubi-server/gen
 kyuubi-server/src/main/antlr4/org/apache/kyuubi/sql/*.tokens
 kyuubi-server/src/main/antlr4/org/apache/kyuubi/sql/gen/
+
+# For draw.io
+.$*.bkp
+.$*.dtmp
diff --git a/LICENSE-binary b/LICENSE-binary
@@ -214,7 +214,7 @@ commons-logging:commons-logging
 org.apache.commons:commons-lang3
 org.apache.derby:derby
 com.google.errorprone:error_prone_annotations
-net.jodah:failsafe
+dev.failsafe:failsafe
 com.jakewharton.fliptables:fliptables
 com.github.mifmif:generex
 io.grpc:grpc-api
@@ -225,6 +225,7 @@ io.grpc:grpc-netty
 io.grpc:grpc-protobuf-lite
 io.grpc:grpc-protobuf
 io.grpc:grpc-stub
+io.grpc:grpc-util
 com.google.code.gson:gson
 com.google.guava:failureaccess
 com.google.guava:guava