-
Notifications
You must be signed in to change notification settings - Fork 919
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[KYUUBI #5867] HiveEngine support run on YARN mode
# 🔍 Description ## Issue References 🔗 This PR aims to support hive engine run on yarn mode, close #5867 ## Describe Your Solution 🔧 Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. List any dependencies that are required for this change. ## Types of changes 🔖 - [ ] Bugfix (non-breaking change which fixes an issue) - [x] New feature (non-breaking change which adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to change) ## Test Plan 🧪 #### Behavior Without This Pull Request ⚰️ #### Behavior With This Pull Request 🎉 #### Related Unit Tests --- # Checklists ## 📝 Author Self Checklist - [ ] My code follows the [style guidelines](https://kyuubi.readthedocs.io/en/master/contributing/code/style.html) of this project - [ ] I have performed a self-review - [ ] I have commented my code, particularly in hard-to-understand areas - [ ] I have made corresponding changes to the documentation - [ ] My changes generate no new warnings - [ ] I have added tests that prove my fix is effective or that my feature works - [ ] New and existing unit tests pass locally with my changes - [x] This patch was not authored or co-authored using [Generative Tooling](https://www.apache.org/legal/generative-tooling.html) ## 📝 Committer Pre-Merge Checklist - [x] Pull request title is okay. - [x] No license issues. - [x] Milestone correctly set? - [x] Test coverage is ok - [x] Assignees are selected. - [x] Minimum number of approvals - [x] No changes are requested **Be nice. Be informative.** Closes #5868 from Yikf/hive-on-yarn. Closes #5867 44f7287 [yikaifei] fix 3c17d2c [yikaifei] fix test 5474ebf [yikaifei] parse classpath 6b97c42 [Cheng Pan] Update kyuubi-common/src/main/scala/org/apache/kyuubi/engine/deploy/yarn/EngineYarnModeSubmitter.scala 34a67b4 [Cheng Pan] Update kyuubi-common/src/main/scala/org/apache/kyuubi/engine/deploy/yarn/EngineYarnModeSubmitter.scala 5e5045e [yikaifei] fix app type d1eb5ae [yikaifei] fix d89d09c [Cheng Pan] Update kyuubi-common/src/main/scala/org/apache/kyuubi/engine/deploy/yarn/EngineYarnModeSubmitter.scala 1fa18ba [Cheng Pan] Update kyuubi-common/src/main/scala/org/apache/kyuubi/engine/deploy/yarn/ApplicationMaster.scala 1b0b77f [Cheng Pan] Update kyuubi-common/src/main/scala/org/apache/kyuubi/engine/deploy/yarn/ApplicationMaster.scala 2ed1d44 [Cheng Pan] Update kyuubi-common/src/main/scala/org/apache/kyuubi/engine/deploy/yarn/EngineYarnModeSubmitter.scala 98ff19c [yikaifei] HiveEngine support run on YARN mode Lead-authored-by: yikaifei <[email protected]> Co-authored-by: Cheng Pan <[email protected]> Signed-off-by: Cheng Pan <[email protected]>
- Loading branch information
Showing
24 changed files
with
1,412 additions
and
15 deletions.
There are no files selected for viewing
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
65 changes: 65 additions & 0 deletions
65
...ql-engine/src/main/scala/org/apache/kyuubi/engine/hive/deploy/HiveYarnModeSubmitter.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,65 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one or more | ||
| * contributor license agreements. See the NOTICE file distributed with | ||
| * this work for additional information regarding copyright ownership. | ||
| * The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| * (the "License"); you may not use this file except in compliance with | ||
| * the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
| package org.apache.kyuubi.engine.hive.deploy | ||
|
|
||
| import java.io.File | ||
|
|
||
| import scala.collection.mutable.ListBuffer | ||
|
|
||
| import org.apache.kyuubi.Utils | ||
| import org.apache.kyuubi.config.KyuubiConf.ENGINE_HIVE_EXTRA_CLASSPATH | ||
| import org.apache.kyuubi.engine.deploy.yarn.EngineYarnModeSubmitter | ||
| import org.apache.kyuubi.engine.hive.HiveSQLEngine | ||
|
|
||
| object HiveYarnModeSubmitter extends EngineYarnModeSubmitter { | ||
|
|
||
| def main(args: Array[String]): Unit = { | ||
| Utils.fromCommandLineArgs(args, kyuubiConf) | ||
| submitApplication() | ||
| } | ||
|
|
||
| override var engineType: String = "hive" | ||
|
|
||
| override def engineMainClass(): String = HiveSQLEngine.getClass.getName | ||
|
|
||
| /** | ||
| * Jar list for the Hive engine. | ||
| */ | ||
| override def engineExtraJars(): Seq[File] = { | ||
| val hadoopCp = sys.env.get("HIVE_HADOOP_CLASSPATH") | ||
| val extraCp = kyuubiConf.get(ENGINE_HIVE_EXTRA_CLASSPATH) | ||
| val jars = new ListBuffer[File] | ||
| hadoopCp.foreach(cp => parseClasspath(cp, jars)) | ||
| extraCp.foreach(cp => parseClasspath(cp, jars)) | ||
| jars.toSeq | ||
| } | ||
|
|
||
| private[hive] def parseClasspath(classpath: String, jars: ListBuffer[File]): Unit = { | ||
| classpath.split(":").filter(_.nonEmpty).foreach { cp => | ||
| if (cp.endsWith("/*")) { | ||
| val dir = cp.substring(0, cp.length - 2) | ||
| new File(dir) match { | ||
| case f if f.isDirectory => | ||
| f.listFiles().filter(_.getName.endsWith(".jar")).foreach(jars += _) | ||
| case _ => | ||
| } | ||
| } else { | ||
| jars += new File(cp) | ||
| } | ||
| } | ||
| } | ||
| } |
39 changes: 39 additions & 0 deletions
39
...gine/src/test/scala/org/apache/kyuubi/engine/hive/deploy/HiveYarnModeSubmitterSuite.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,39 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one or more | ||
| * contributor license agreements. See the NOTICE file distributed with | ||
| * this work for additional information regarding copyright ownership. | ||
| * The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| * (the "License"); you may not use this file except in compliance with | ||
| * the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
| package org.apache.kyuubi.engine.hive.deploy | ||
|
|
||
| import java.io.File | ||
|
|
||
| import scala.collection.mutable.ListBuffer | ||
|
|
||
| import org.apache.kyuubi.{KYUUBI_VERSION, KyuubiFunSuite, SCALA_COMPILE_VERSION, Utils} | ||
|
|
||
| class HiveYarnModeSubmitterSuite extends KyuubiFunSuite { | ||
| val hiveEngineHome: String = Utils.getCodeSourceLocation(getClass).split("/target")(0) | ||
|
|
||
| test("hadoop class path") { | ||
| val jars = new ListBuffer[File] | ||
| val classpath = | ||
| s"$hiveEngineHome/target/scala-$SCALA_COMPILE_VERSION/jars/*:" + | ||
| s"$hiveEngineHome/target/kyuubi-hive-sql-engine-$SCALA_COMPILE_VERSION-$KYUUBI_VERSION.jar" | ||
| HiveYarnModeSubmitter.parseClasspath(classpath, jars) | ||
| assert(jars.nonEmpty) | ||
| assert(jars.exists( | ||
| _.getName == s"kyuubi-hive-sql-engine-$SCALA_COMPILE_VERSION-$KYUUBI_VERSION.jar")) | ||
| } | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
47 changes: 47 additions & 0 deletions
47
...st/scala/org/apache/kyuubi/it/hive/operation/KyuubiOperationHiveEngineYarnModeSuite.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,47 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one or more | ||
| * contributor license agreements. See the NOTICE file distributed with | ||
| * this work for additional information regarding copyright ownership. | ||
| * The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| * (the "License"); you may not use this file except in compliance with | ||
| * the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
| package org.apache.kyuubi.it.hive.operation | ||
|
|
||
| import org.apache.kyuubi.{HiveEngineTests, Utils, WithKyuubiServerAndHadoopMiniCluster} | ||
| import org.apache.kyuubi.config.KyuubiConf | ||
| import org.apache.kyuubi.config.KyuubiConf.{ENGINE_IDLE_TIMEOUT, ENGINE_TYPE, KYUUBI_ENGINE_ENV_PREFIX, KYUUBI_HOME} | ||
| import org.apache.kyuubi.engine.deploy.DeployMode | ||
|
|
||
| class KyuubiOperationHiveEngineYarnModeSuite extends HiveEngineTests | ||
| with WithKyuubiServerAndHadoopMiniCluster { | ||
|
|
||
| override protected val conf: KyuubiConf = { | ||
| val metastore = Utils.createTempDir(prefix = getClass.getSimpleName) | ||
| metastore.toFile.delete() | ||
| KyuubiConf() | ||
| .set(s"$KYUUBI_ENGINE_ENV_PREFIX.$KYUUBI_HOME", kyuubiHome) | ||
| .set(ENGINE_TYPE, "HIVE_SQL") | ||
| .set(KyuubiConf.ENGINE_HIVE_DEPLOY_MODE, DeployMode.YARN.toString) | ||
| // increase this to 30s as hive session state and metastore client is slow initializing | ||
| .setIfMissing(ENGINE_IDLE_TIMEOUT, 30000L) | ||
| .set("javax.jdo.option.ConnectionURL", s"jdbc:derby:;databaseName=$metastore;create=true") | ||
| } | ||
|
|
||
| override def beforeAll(): Unit = { | ||
| super.beforeAll() | ||
| conf | ||
| .set(KyuubiConf.ENGINE_DEPLOY_YARN_MODE_MEMORY, Math.min(getYarnMaximumAllocationMb, 1024)) | ||
| .set(KyuubiConf.ENGINE_DEPLOY_YARN_MODE_CORES, 1) | ||
| } | ||
|
|
||
| override protected def jdbcUrl: String = getJdbcUrl | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
34 changes: 34 additions & 0 deletions
34
kyuubi-common/src/main/scala/org/apache/kyuubi/engine/deploy/DeployMode.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,34 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one or more | ||
| * contributor license agreements. See the NOTICE file distributed with | ||
| * this work for additional information regarding copyright ownership. | ||
| * The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| * (the "License"); you may not use this file except in compliance with | ||
| * the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
| package org.apache.kyuubi.engine.deploy | ||
|
|
||
| object DeployMode extends Enumeration { | ||
| type DeployMode = Value | ||
| val | ||
| /** | ||
| * In this mode, the engine will be launched locally. | ||
| */ | ||
| LOCAL, | ||
| /** | ||
| * In this mode, the engine will be launched on YARN. | ||
| */ | ||
| YARN, | ||
| /** | ||
| * In this mode, the engine will be launched on Kubernetes. | ||
| */ | ||
| KUBERNETES = Value | ||
| } |
Oops, something went wrong.