diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/ProcBuilder.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/ProcBuilder.scala index 84807a62d87..9c7f0f510cb 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/ProcBuilder.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/ProcBuilder.scala @@ -287,10 +287,10 @@ trait ProcBuilder { override def toString: String = { if (commands == null) { - super.toString() + super.toString } else { Utils.redactCommandLineArgs(conf, commands).map { - case arg if arg.startsWith("--") => s"\\\n\t$arg" + case arg if arg.startsWith("-") => s"\\\n\t$arg" case arg => arg }.mkString(" ") } diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/chat/ChatProcessBuilder.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/chat/ChatProcessBuilder.scala index 3e4a20de373..121a20f5f23 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/chat/ChatProcessBuilder.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/chat/ChatProcessBuilder.scala @@ -103,13 +103,15 @@ class ChatProcessBuilder( override def toString: String = { if (commands == null) { - super.toString() + super.toString } else { Utils.redactCommandLineArgs(conf, commands).map { - case arg if arg.startsWith("-") || arg == mainClass => s"\\\n\t$arg" case arg if arg.contains(ENGINE_CHAT_GPT_API_KEY.key) => s"${ENGINE_CHAT_GPT_API_KEY.key}=$REDACTION_REPLACEMENT_TEXT" case arg => arg + }.map { + case arg if arg.startsWith("-") || arg == mainClass => s"\\\n\t$arg" + case arg => arg }.mkString(" ") } } diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/hive/HiveProcessBuilder.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/hive/HiveProcessBuilder.scala index 61fe55887ea..3325d5f2e02 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/hive/HiveProcessBuilder.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/hive/HiveProcessBuilder.scala @@ -116,8 +116,6 @@ class HiveProcessBuilder( buffer.toArray } - override def toString: String = Utils.redactCommandLineArgs(conf, commands).mkString("\n") - override def shortName: String = "hive" } diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/jdbc/JdbcProcessBuilder.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/jdbc/JdbcProcessBuilder.scala index 14ad53b20a8..3849c6431e9 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/jdbc/JdbcProcessBuilder.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/jdbc/JdbcProcessBuilder.scala @@ -106,13 +106,16 @@ class JdbcProcessBuilder( override def toString: String = { if (commands == null) { - super.toString() + super.toString } else { Utils.redactCommandLineArgs(conf, commands).map { case arg if arg.contains(ENGINE_JDBC_CONNECTION_PASSWORD.key) => s"${ENGINE_JDBC_CONNECTION_PASSWORD.key}=$REDACTION_REPLACEMENT_TEXT" case arg => arg - }.mkString("\n") + }.map { + case arg if arg.startsWith("-") => s"\\\n\t$arg" + case arg => arg + }.mkString(" ") } } } diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/trino/TrinoProcessBuilder.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/trino/TrinoProcessBuilder.scala index 041219dd0fb..5b755dec5a1 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/trino/TrinoProcessBuilder.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/trino/TrinoProcessBuilder.scala @@ -111,7 +111,7 @@ class TrinoProcessBuilder( override def toString: String = { if (commands == null) { - super.toString() + super.toString } else { Utils.redactCommandLineArgs(conf, commands).map { case arg if arg.contains(ENGINE_TRINO_CONNECTION_PASSWORD.key) => @@ -121,7 +121,10 @@ class TrinoProcessBuilder( case arg if arg.contains(ENGINE_TRINO_CONNECTION_TRUSTSTORE_PASSWORD.key) => s"${ENGINE_TRINO_CONNECTION_TRUSTSTORE_PASSWORD.key}=$REDACTION_REPLACEMENT_TEXT" case arg => arg - }.mkString("\n") + }.map { + case arg if arg.startsWith("-") => s"\\\n\t$arg" + case arg => arg + }.mkString(" ") } } } diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/flink/FlinkProcessBuilderSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/flink/FlinkProcessBuilderSuite.scala index 26e355a87bd..84be010ed4b 100644 --- a/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/flink/FlinkProcessBuilderSuite.scala +++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/flink/FlinkProcessBuilderSuite.scala @@ -81,8 +81,11 @@ class FlinkProcessBuilderSuite extends KyuubiFunSuite { val actualCommands = builder.toString val classpathStr = constructClasspathStr(builder) val expectedCommands = - s"$javaPath -Xmx512m -agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5005 " + - s"-cp $classpathStr $mainClassStr \\\\\\n\\t--conf kyuubi.session.user=vinoyang $confStr" + s"""$javaPath \\\\ + |\\t-Xmx512m \\\\ + |\\t-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5005 \\\\ + |\\t-cp $classpathStr $mainClassStr \\\\ + |\\t--conf kyuubi.session.user=vinoyang $confStr""".stripMargin val regex = new Regex(expectedCommands) val matcher = regex.pattern.matcher(actualCommands) assert(matcher.matches()) @@ -90,19 +93,20 @@ class FlinkProcessBuilderSuite extends KyuubiFunSuite { private def matchActualAndExpectedApplicationMode(builder: FlinkProcessBuilder): Unit = { val actualCommands = builder.toString + // scalastyle:off line.size.limit val expectedCommands = - escapePaths(s"${builder.flinkExecutable} run-application ") + - s"-t yarn-application " + - s"-Dyarn.ship-files=.*\\/flink-sql-client.*jar;.*\\/flink-sql-gateway.*jar;$tempUdfJar" + - s";.*\\/hive-site\\.xml " + - s"-Dyarn\\.application\\.name=kyuubi_.* " + - s"-Dyarn\\.tags=KYUUBI " + - s"-Dcontainerized\\.master\\.env\\.FLINK_CONF_DIR=\\. " + - s"-Dcontainerized\\.master\\.env\\.HIVE_CONF_DIR=\\. " + - s"-Dexecution.target=yarn-application " + - s"-c org\\.apache\\.kyuubi\\.engine\\.flink\\.FlinkSQLEngine " + - s".*kyuubi-flink-sql-engine_.*jar" + - s"(?: \\\\\\n\\t--conf \\S+=\\S+)+" + escapePaths( + s"""${builder.flinkExecutable} run-application \\\\ + |\\t-t yarn-application \\\\ + |\\t-Dyarn.ship-files=.*flink-sql-client.*jar;.*flink-sql-gateway.*jar;$tempUdfJar;.*hive-site.xml \\\\ + |\\t-Dyarn.application.name=kyuubi_.* \\\\ + |\\t-Dyarn.tags=KYUUBI \\\\ + |\\t-Dcontainerized.master.env.FLINK_CONF_DIR=. \\\\ + |\\t-Dcontainerized.master.env.HIVE_CONF_DIR=. \\\\ + |\\t-Dexecution.target=yarn-application \\\\ + |\\t-c org.apache.kyuubi.engine.flink.FlinkSQLEngine .*kyuubi-flink-sql-engine_.*jar""".stripMargin + + "(?: \\\\\\n\\t--conf \\S+=\\S+)+") + // scalastyle:on line.size.limit val regex = new Regex(expectedCommands) val matcher = regex.pattern.matcher(actualCommands) assert(matcher.matches()) diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/hive/HiveProcessBuilderSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/hive/HiveProcessBuilderSuite.scala index bb9884dfa4b..a2f39633ca4 100644 --- a/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/hive/HiveProcessBuilderSuite.scala +++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/hive/HiveProcessBuilderSuite.scala @@ -30,18 +30,18 @@ class HiveProcessBuilderSuite extends KyuubiFunSuite { override def env: Map[String, String] = super.env + (HIVE_HADOOP_CLASSPATH_KEY -> "/hadoop") } val commands = builder.toString.split('\n') - assert(commands.head.endsWith("bin/java"), "wrong exec") - assert(builder.toString.contains("--conf\nkyuubi.session.user=kyuubi")) + assert(commands.head.contains("bin/java"), "wrong exec") + assert(builder.toString.contains("--conf kyuubi.session.user=kyuubi")) assert(commands.exists(ss => ss.contains("kyuubi-hive-sql-engine")), "wrong classpath") - assert(builder.toString.contains("--conf\nkyuubi.on=off")) + assert(builder.toString.contains("--conf kyuubi.on=off")) } test("default engine memory") { val conf = KyuubiConf() .set(ENGINE_HIVE_EXTRA_CLASSPATH, "/hadoop") val builder = new HiveProcessBuilder("kyuubi", conf) - val commands = builder.toString.split('\n') - assert(commands.contains("-Xmx1g")) + val command = builder.toString + assert(command.contains("-Xmx1g")) } test("set engine memory") { @@ -49,8 +49,8 @@ class HiveProcessBuilderSuite extends KyuubiFunSuite { .set(ENGINE_HIVE_MEMORY, "5g") .set(ENGINE_HIVE_EXTRA_CLASSPATH, "/hadoop") val builder = new HiveProcessBuilder("kyuubi", conf) - val commands = builder.toString.split('\n') - assert(commands.contains("-Xmx5g")) + val command = builder.toString + assert(command.contains("-Xmx5g")) } test("set engine java opts") { @@ -60,8 +60,8 @@ class HiveProcessBuilderSuite extends KyuubiFunSuite { ENGINE_HIVE_JAVA_OPTIONS, "-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5005") val builder = new HiveProcessBuilder("kyuubi", conf) - val commands = builder.toString.split('\n') - assert(commands.contains("-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5005")) + val command = builder.toString + assert(command.contains("-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5005")) } test("set engine extra classpath") { diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/jdbc/JdbcProcessBuilderSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/jdbc/JdbcProcessBuilderSuite.scala index f85e363d39e..2be39d0f319 100644 --- a/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/jdbc/JdbcProcessBuilderSuite.scala +++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/jdbc/JdbcProcessBuilderSuite.scala @@ -27,13 +27,13 @@ class JdbcProcessBuilderSuite extends KyuubiFunSuite { .set(ENGINE_JDBC_CONNECTION_URL.key, "") .set(ENGINE_JDBC_CONNECTION_PASSWORD.key, "123456") val builder = new JdbcProcessBuilder("kyuubi", conf) - val commands = builder.toString.split("\n") - assert(commands.head.endsWith("bin/java"), "wrong exec") - assert(builder.toString.contains("--conf\nkyuubi.session.user=kyuubi")) - assert(commands.exists(ss => ss.contains("kyuubi-jdbc-engine")), "wrong classpath") - assert(builder.toString.contains("--conf\nkyuubi.on=off")) - assert(builder.toString.contains( - "--conf\nkyuubi.engine.jdbc.connection.password=*********(redacted)")) + val command = builder.toString + assert(command.contains("bin/java"), "wrong exec") + assert(command.contains("--conf kyuubi.session.user=kyuubi")) + assert(command.contains("kyuubi-jdbc-engine"), "wrong classpath") + assert(command.contains("--conf kyuubi.on=off")) + assert(command.contains( + "--conf kyuubi.engine.jdbc.connection.password=*********(redacted)")) } test("capture error from jdbc process builder") { @@ -47,8 +47,8 @@ class JdbcProcessBuilderSuite extends KyuubiFunSuite { val conf = KyuubiConf() .set(ENGINE_JDBC_CONNECTION_URL.key, "") val builder = new JdbcProcessBuilder("kyuubi", conf) - val commands = builder.toString.split("\n") - assert(commands.contains("-Xmx1g")) + val command = builder.toString + assert(command.contains("-Xmx1g")) } test("set engine memory") { @@ -56,8 +56,8 @@ class JdbcProcessBuilderSuite extends KyuubiFunSuite { .set(ENGINE_JDBC_MEMORY, "5g") .set(ENGINE_JDBC_CONNECTION_URL.key, "") val builder = new JdbcProcessBuilder("kyuubi", conf) - val commands = builder.toString.split("\n") - assert(commands.contains("-Xmx5g")) + val command = builder.toString + assert(command.contains("-Xmx5g")) } test("set engine java options") { @@ -67,8 +67,8 @@ class JdbcProcessBuilderSuite extends KyuubiFunSuite { ENGINE_JDBC_JAVA_OPTIONS, "-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5005") val builder = new JdbcProcessBuilder("kyuubi", conf) - val commands = builder.toString.split("\n") - assert(commands.contains("-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5005")) + val command = builder.toString + assert(command.contains("-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5005")) } test("set extra classpath") { @@ -76,7 +76,7 @@ class JdbcProcessBuilderSuite extends KyuubiFunSuite { .set(ENGINE_JDBC_CONNECTION_URL.key, "") .set(ENGINE_JDBC_EXTRA_CLASSPATH, "/dummy_classpath/*") val builder = new JdbcProcessBuilder("kyuubi", conf) - val commands = builder.toString - assert(commands.contains("/dummy_classpath/*")) + val command = builder.toString + assert(command.contains("/dummy_classpath/*")) } } diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/trino/TrinoProcessBuilderSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/trino/TrinoProcessBuilderSuite.scala index 2c37c41bc4b..a4dfad186a1 100644 --- a/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/trino/TrinoProcessBuilderSuite.scala +++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/trino/TrinoProcessBuilderSuite.scala @@ -30,11 +30,11 @@ class TrinoProcessBuilderSuite extends KyuubiFunSuite { .set(ENGINE_TRINO_CONNECTION_CATALOG, "dummy_catalog") val builder = new TrinoProcessBuilder("kyuubi", conf) val commands = builder.toString.split("\n") - assert(commands.head.endsWith("java")) - assert(builder.toString.contains(s"--conf\n${KYUUBI_SESSION_USER_KEY}=kyuubi")) - assert(builder.toString.contains(s"--conf\n${ENGINE_TRINO_CONNECTION_URL.key}=dummy_url")) + assert(commands.head.contains("java")) + assert(builder.toString.contains(s"--conf ${KYUUBI_SESSION_USER_KEY}=kyuubi")) + assert(builder.toString.contains(s"--conf ${ENGINE_TRINO_CONNECTION_URL.key}=dummy_url")) assert(builder.toString.contains( - s"--conf\n${ENGINE_TRINO_CONNECTION_CATALOG.key}=dummy_catalog")) + s"--conf ${ENGINE_TRINO_CONNECTION_CATALOG.key}=dummy_catalog")) } test("capture error from trino process builder") { @@ -49,8 +49,8 @@ class TrinoProcessBuilderSuite extends KyuubiFunSuite { .set(ENGINE_TRINO_CONNECTION_URL, "dummy_url") .set(ENGINE_TRINO_CONNECTION_CATALOG, "dummy_catalog") val builder = new TrinoProcessBuilder("kyuubi", conf) - val commands = builder.toString.split("\n") - assert(commands.contains("-Xmx1g")) + val command = builder.toString + assert(command.contains("-Xmx1g")) } test("set engine memory") { @@ -59,8 +59,8 @@ class TrinoProcessBuilderSuite extends KyuubiFunSuite { .set(ENGINE_TRINO_CONNECTION_CATALOG, "dummy_catalog") .set(ENGINE_TRINO_MEMORY, "5g") val builder = new TrinoProcessBuilder("kyuubi", conf) - val commands = builder.toString.split("\n") - assert(commands.contains("-Xmx5g")) + val command = builder.toString + assert(command.contains("-Xmx5g")) } test("set engine java options") { @@ -71,8 +71,8 @@ class TrinoProcessBuilderSuite extends KyuubiFunSuite { ENGINE_TRINO_JAVA_OPTIONS, "-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5005") val builder = new TrinoProcessBuilder("kyuubi", conf) - val commands = builder.toString.split("\n") - assert(commands.contains("-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5005")) + val command = builder.toString + assert(command.contains("-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5005")) } test("set extra classpath") {