Merge branch 'master' into KYUUBI-5284

.asf.yaml

@@ -30,6 +30,7 @@ github:
     - hive
     - sql
     - kubernetes
+    - hacktoberfest
   enabled_merge_buttons:
     squash: true
     merge: false
@@ -39,7 +40,7 @@ github:
     issues: true
     discussions: true
     wiki: false
-    projects: false
+    projects: true
 notifications:
   commits: [email protected]
   issues: [email protected]

.gitattributes

@@ -26,6 +26,7 @@ NOTICE-binary export-ignore
 *.bat text eol=crlf
 *.cmd text eol=crlf
 *.java text eol=lf
+*.md text eol=lf
 *.scala text eol=lf
 *.xml text eol=lf
 *.py text eol=lf

.github/ISSUE_TEMPLATE/code-contrib-task.yml

@@ -0,0 +1,115 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# This is a dedicated issue template for 2023 Kyuubi Code Contribution Program, all proposed
+# tasks will be listed at https://github.com/orgs/apache/projects/296 after approval
+#
+name: 2023 Kyuubi Code Contribution Task
+title: "[TASK][<LEVEL>] <The Task Title>"
+description: Propose a task for 2023 Kyuubi Code Contribution Program
+labels: [ "hacktoberfest" ]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        You are very welcome to propose new task for 2023 Kyuubi Code Contribution Program.
+        Your brilliant ideas keep Apache Kyuubi evolving.
+        Please replace the placeholder `<LEVEL>` in the issue title with one of the following options:
+        - TRIVIAL - it's usually for new contributors to learn the contributor process, e.g. how to cut branch,
+          how to use GitHub to send PR, how to response with reviewers, the contributor should not stay at this
+          stage too long.
+        - EASY - tasks like minor bugs, or simple features without requirements of knowledge for whole Kyuubi
+          architecture.
+        - MEDIUM - tasks typical requires that contributors have knowledge on one or more Kyuubi components,
+          normally, unit tests and integration tests is also required to verify the implementations.
+        - CHALLENGE - tasks requires that contributors have deep knowledge on one or more Kyuubi components,
+          have good logical thinking and the ability to solve complex problems, be proficient in programming
+          skills or algorithms
+
+  - type: checkboxes
+    attributes:
+      label: Code of Conduct
+      description: The Code of Conduct helps create a safe space for everyone. We require that everyone agrees to it.
+      options:
+        - label: >
+            I agree to follow this project's [Code of Conduct](https://www.apache.org/foundation/policies/conduct)
+          required: true
+
+  - type: checkboxes
+    attributes:
+      label: Search before creating
+      options:
+        - label: >
+            I have searched in the [task list](https://github.com/orgs/apache/projects/296) and found no similar
+            tasks.
+          required: true
+
+  - type: checkboxes
+    attributes:
+      label: Mentor
+      description: Mentor is required for MEDIUM and CHALLENGE tasks, to guide contributors to complete the task.
+      options:
+        - label: >
+            I have sufficient knowledge and experience of this task, and I volunteer to be the mentor of this task
+            to guide contributors to complete the task.
+          required: false
+
+  - type: textarea
+    attributes:
+      label: Skill requirements
+      description: Which stills are required for contributors who want to take this task?
+      placeholder: |
+        e.g.
+        - Basic knowledge on Scala Programing Language
+        - Familiar with Apache Maven, Docker and GitHub Action
+        - Basic knowledge on network programing and Apache Thrift RPC framework
+        - Familiar with Apache Spark
+        - ...
+    validations:
+      required: true
+
+  - type: textarea
+    attributes:
+      label: Background and Goals
+      description: What's the current problem, and what's the final status should be after the task is completed?
+      placeholder: >
+        Please describe the background and your goal for requesting this task.
+    validations:
+      required: true
+
+  - type: textarea
+    attributes:
+      label: Implementation steps
+      description: How could it be implemented?
+      placeholder: >
+        Please list the implementation steps in as much detail as possible so that contributors who meet
+        the skill requirements could complete the task quickly and independently.
+    validations:
+      required: true
+
+  - type: textarea
+    attributes:
+      label: Additional context
+      placeholder: >
+        Anything else that related to this task that the contributors need to know.
+    validations:
+      required: false
+
+  - type: markdown
+    attributes:
+      value: "Thanks for taking the time to fill out this task form!"

.github/workflows/license.yml

@@ -45,7 +45,7 @@ jobs:
       - run: >-
           build/mvn org.apache.rat:apache-rat-plugin:check
           -Ptpcds -Pspark-block-cleaner -Pkubernetes-it
-          -Pspark-3.1 -Pspark-3.2 -Pspark-3.3 -Pspark-3.4
+          -Pspark-3.1 -Pspark-3.2 -Pspark-3.3 -Pspark-3.4 -Pspark-3.5
       - name: Upload rat report
         if: failure()
         uses: actions/upload-artifact@v3

.github/workflows/master.yml

@@ -52,6 +52,7 @@ jobs:
           - '3.2'
           - '3.3'
           - '3.4'
+          - '3.5'
         spark-archive: [""]
         exclude-tags: [""]
         comment: ["normal"]
@@ -71,6 +72,11 @@ jobs:
             spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.3.3 -Dspark.archive.name=spark-3.3.3-bin-hadoop3.tgz -Pzookeeper-3.6'
             exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.HoodieTest,org.apache.kyuubi.tags.SparkLocalClusterTest'
             comment: 'verify-on-spark-3.3-binary'
+          - java: 8
+            spark: '3.4'
+            spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.5.0 -Dspark.archive.name=spark-3.5.0-bin-hadoop3.tgz -Pzookeeper-3.6'
+            exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.SparkLocalClusterTest'
+            comment: 'verify-on-spark-3.5-binary'
         exclude:
           # SPARK-33772: Spark supports JDK 17 since 3.3.0
           - java: 17
@@ -164,16 +170,18 @@ jobs:
             **/target/unit-tests.log
             **/kyuubi-spark-sql-engine.log*
 
-  scala213:
-    name: Scala Compilation Test
+  scala-test:
+    name: Scala Test
     runs-on: ubuntu-22.04
     strategy:
       fail-fast: false
       matrix:
-        java:
-          - '8'
         scala:
           - '2.13'
+        java:
+          - '8'
+        spark:
+          - '3.4'
     steps:
       - uses: actions/checkout@v3
       - name: Tune Runner VM
@@ -187,14 +195,24 @@ jobs:
           check-latest: false
       - name: Setup Maven
         uses: ./.github/actions/setup-maven
+      - name: Cache Engine Archives
+        uses: ./.github/actions/cache-engine-archives
       - name: Build on Scala ${{ matrix.scala }}
         run: |
-          MODULES='!externals/kyuubi-flink-sql-engine'
-          ./build/mvn clean install -pl ${MODULES} -am \
-          -DskipTests -Pflink-provided,hive-provided,spark-provided \
-          -Pjava-${{ matrix.java }} \
-          -Pscala-${{ matrix.scala }} \
-          -Pspark-3.3
+          TEST_MODULES="!externals/kyuubi-flink-sql-engine,!integration-tests/kyuubi-flink-it"
+          ./build/mvn clean install ${MVN_OPT} -pl ${TEST_MODULES} -am \
+          -Pscala-${{ matrix.scala }} -Pjava-${{ matrix.java }} -Pspark-${{ matrix.spark }}
+      - name: Upload test logs
+        if: failure()
+        uses: actions/upload-artifact@v3
+        with:
+          name: unit-tests-log-scala-${{ matrix.scala }}-java-${{ matrix.java }}-spark-${{ matrix.spark }}
+          path: |
+            **/target/unit-tests.log
+            **/kyuubi-spark-sql-engine.log*
+            **/kyuubi-spark-batch-submit.log*
+            **/kyuubi-jdbc-engine.log*
+            **/kyuubi-hive-sql-engine.log*
 
   flink-it:
     name: Flink Test

.github/workflows/style.yml

@@ -69,6 +69,7 @@ jobs:
           build/mvn clean install ${MVN_OPT} -pl extensions/spark/kyuubi-extension-spark-3-1 -Pspark-3.1
           build/mvn clean install ${MVN_OPT} -pl extensions/spark/kyuubi-extension-spark-3-3,extensions/spark/kyuubi-spark-connector-hive -Pspark-3.3
           build/mvn clean install ${MVN_OPT} -pl extensions/spark/kyuubi-extension-spark-3-4 -Pspark-3.4
+          build/mvn clean install ${MVN_OPT} -pl extensions/spark/kyuubi-extension-spark-3-5 -Pspark-3.5
 
       - name: Scalastyle with maven
         id: scalastyle-check

build/dist

@@ -335,7 +335,7 @@ if [[ -f "$KYUUBI_HOME/tools/spark-block-cleaner/target/spark-block-cleaner_${SC
 fi
 
 # Copy Kyuubi Spark extension
-SPARK_EXTENSION_VERSIONS=('3-1' '3-2' '3-3' '3-4')
+SPARK_EXTENSION_VERSIONS=('3-1' '3-2' '3-3' '3-4' '3-5')
 # shellcheck disable=SC2068
 for SPARK_EXTENSION_VERSION in ${SPARK_EXTENSION_VERSIONS[@]}; do
   if [[ -f $"$KYUUBI_HOME/extensions/spark/kyuubi-extension-spark-$SPARK_EXTENSION_VERSION/target/kyuubi-extension-spark-${SPARK_EXTENSION_VERSION}_${SCALA_VERSION}-${VERSION}.jar" ]]; then

build/release/release.sh

@@ -52,6 +52,21 @@ if [[ ${RELEASE_VERSION} =~ .*-SNAPSHOT ]]; then
   exit 1
 fi
 
+if [ -n "${JAVA_HOME}" ]; then
+  JAVA="${JAVA_HOME}/bin/java"
+elif [ "$(command -v java)" ]; then
+  JAVA="java"
+else
+  echo "JAVA_HOME is not set" >&2
+  exit 1
+fi
+
+JAVA_VERSION=$($JAVA -version 2>&1 | awk -F '"' '/version/ {print $2}')
+if [[ $JAVA_VERSION != 1.8.* ]]; then
+  echo "Unexpected Java version: $JAVA_VERSION. Java 8 is required for release."
+  exit 1
+fi
+
 RELEASE_TAG="v${RELEASE_VERSION}-rc${RELEASE_RC_NO}"
 
 SVN_STAGING_REPO="https://dist.apache.org/repos/dist/dev/kyuubi"
@@ -101,6 +116,9 @@ upload_nexus_staging() {
     -s "${KYUUBI_DIR}/build/release/asf-settings.xml" \
     -pl extensions/spark/kyuubi-extension-spark-3-2 -am
   ${KYUUBI_DIR}/build/mvn clean deploy -DskipTests -Papache-release,flink-provided,spark-provided,hive-provided,spark-3.3 \
+    -s "${KYUUBI_DIR}/build/release/asf-settings.xml" \
+    -pl extensions/spark/kyuubi-extension-spark-3-3 -am
+  ${KYUUBI_DIR}/build/mvn clean deploy -DskipTests -Papache-release,flink-provided,spark-provided,hive-provided,spark-3.4 \
     -s "${KYUUBI_DIR}/build/release/asf-settings.xml"
 }
 

charts/kyuubi/Chart.yaml

@@ -20,7 +20,7 @@ name: kyuubi
 description: A Helm chart for Kyuubi server
 type: application
 version: 0.1.0
-appVersion: 1.7.1
+appVersion: 1.7.3
 home: https://kyuubi.apache.org
 icon: https://raw.githubusercontent.com/apache/kyuubi/master/docs/imgs/logo.png
 sources:

dev/dependencyList

@@ -183,7 +183,7 @@ simpleclient_tracer_otel/0.16.0//simpleclient_tracer_otel-0.16.0.jar
 simpleclient_tracer_otel_agent/0.16.0//simpleclient_tracer_otel_agent-0.16.0.jar
 slf4j-api/1.7.36//slf4j-api-1.7.36.jar
 snakeyaml-engine/2.6//snakeyaml-engine-2.6.jar
-snakeyaml/1.33//snakeyaml-1.33.jar
+snakeyaml/2.2//snakeyaml-2.2.jar
 snappy-java/1.1.8.4//snappy-java-1.1.8.4.jar
 sqlite-jdbc/3.42.0.0//sqlite-jdbc-3.42.0.0.jar
 swagger-annotations/2.2.1//swagger-annotations-2.2.1.jar

dev/kyuubi-codecov/pom.xml

@@ -219,5 +219,15 @@
                 </dependency>
             </dependencies>
         </profile>
+        <profile>
+            <id>spark-3.5</id>
+            <dependencies>
+                <dependency>
+                    <groupId>org.apache.kyuubi</groupId>
+                    <artifactId>kyuubi-extension-spark-3-5_${scala.binary.version}</artifactId>
+                    <version>${project.version}</version>
+                </dependency>
+            </dependencies>
+        </profile>
     </profiles>
 </project>

dev/reformat

@@ -20,7 +20,7 @@ set -x
 
 KYUUBI_HOME="$(cd "`dirname "$0"`/.."; pwd)"
 
-PROFILES="-Pflink-provided,hive-provided,spark-provided,spark-block-cleaner,spark-3.4,spark-3.3,spark-3.2,spark-3.1,tpcds"
+PROFILES="-Pflink-provided,hive-provided,spark-provided,spark-block-cleaner,spark-3.5,spark-3.4,spark-3.3,spark-3.2,spark-3.1,tpcds"
 
 # python style checks rely on `black` in path
 if ! command -v black &> /dev/null

docker/playground/.env

@@ -19,7 +19,7 @@ AWS_JAVA_SDK_VERSION=1.12.367
 HADOOP_VERSION=3.3.6
 HIVE_VERSION=2.3.9
 ICEBERG_VERSION=1.3.1
-KYUUBI_VERSION=1.7.1
+KYUUBI_VERSION=1.7.3
 KYUUBI_HADOOP_VERSION=3.3.5
 POSTGRES_VERSION=12
 POSTGRES_JDBC_VERSION=42.3.4

docs/configuration/settings.md

@@ -460,6 +460,13 @@ You can configure the Kyuubi properties in `$KYUUBI_HOME/conf/kyuubi-defaults.co
 | kyuubi.spnego.keytab    | <undefined> | Keytab file for SPNego principal                                                                                                                                                                                                       | string | 1.6.0 |
 | kyuubi.spnego.principal | <undefined> | SPNego service principal, typical value would look like HTTP/_[email protected]. SPNego service principal would be used when restful Kerberos security is enabled. This needs to be set only if SPNEGO is to be used in authentication. | string | 1.6.0 |
 
+### Yarn
+
+|            Key            | Default |                                                                                                                                                 Meaning                                                                                                                                                 |  Type  | Since |
+|---------------------------|---------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------|-------|
+| kyuubi.yarn.user.admin    | yarn    | When kyuubi.yarn.user.strategy is set to ADMIN, use this admin user to construct YARN client for application management, e.g. kill application.                                                                                                                                                         | string | 1.8.0 |
+| kyuubi.yarn.user.strategy | NONE    | Determine which user to use to construct YARN client for application management, e.g. kill application. Options: <ul><li>NONE: use Kyuubi server user.</li><li>ADMIN: use admin user configured in `kyuubi.yarn.user.admin`.</li><li>OWNER: use session user, typically is application owner.</li></ul> | string | 1.8.0 |
+
 ### Zookeeper
 
 |                       Key                        |      Default       |                                                                                 Meaning                                                                                  |  Type   | Since |

docs/quick_start/quick_start_with_jdbc.md

@@ -35,7 +35,7 @@ The driver is available from Maven Central:
 
 ## Connect to non-kerberized Kyuubi Server
 
-The below java code is using a keytab file to login and connect to Kyuubi server by JDBC.
+The following java code connects directly to the Kyuubi Server by JDBC without using kerberos authentication.
 
 ```java
 package org.apache.kyuubi.examples;
@@ -50,7 +50,7 @@ public class KyuubiJDBC {
   public static void main(String[] args) throws SQLException {
     try (Connection conn = DriverManager.getConnection(kyuubiJdbcUrl)) {
       try (Statement stmt = conn.createStatement()) {
-        try (ResultSet rs = st.executeQuery("show databases")) {
+        try (ResultSet rs = stmt.executeQuery("show databases")) {
           while (rs.next()) {
             System.out.println(rs.getString(1));
           }
@@ -79,11 +79,11 @@ public class KyuubiJDBCDemo {
   public static void main(String[] args) throws SQLException {
     String clientPrincipal = args[0]; // Kerberos principal
     String clientKeytab = args[1];    // Keytab file location
-    String serverPrincipal = arg[2];  // Kerberos principal used by Kyuubi Server
+    String serverPrincipal = args[2]; // Kerberos principal used by Kyuubi Server
     String kyuubiJdbcUrl = String.format(kyuubiJdbcUrlTemplate, clientPrincipal, clientKeytab, serverPrincipal);
     try (Connection conn = DriverManager.getConnection(kyuubiJdbcUrl)) {
       try (Statement stmt = conn.createStatement()) {
-        try (ResultSet rs = st.executeQuery("show databases")) {
+        try (ResultSet rs = stmt.executeQuery("show databases")) {
           while (rs.next()) {
             System.out.println(rs.getString(1));
           }

docs/tools/kyuubi-admin.rst

@@ -99,6 +99,8 @@ Usage: ``bin/kyuubi-admin list engine [options]``
      - The subdomain for the share level of an engine. If not specified, it will read the configuration item kyuubi.engine.share.level.subdomain from kyuubi-defaults.conf.
    * - --hs2ProxyUser
      - The proxy user to impersonate. When specified, it will list engines for the hs2ProxyUser.
+   * - -a --all
+     - All the engine.
 
 .. _list_server: