From 85233640f13b7e3493bfe20d55b11647180add9f Mon Sep 17 00:00:00 2001 From: bkhan Date: Wed, 28 Feb 2024 15:30:29 +0800 Subject: [PATCH 1/4] Display the CPU time consumed by the statement in the Engine tab --- .../src/main/scala/org/apache/spark/ui/EnginePage.scala | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EnginePage.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EnginePage.scala index cae0c03bf4d..aa34bc91239 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EnginePage.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EnginePage.scala @@ -402,6 +402,7 @@ case class EnginePage(parent: EngineTab) extends WebUIPage("") { ("Start Time", true, None), ("Finish Time", true, None), ("Duration", true, None), + ("Cpu Time", true, None), ("Total Statements", true, None)) headerStatRow( @@ -428,6 +429,7 @@ case class EnginePage(parent: EngineTab) extends WebUIPage("") { {formatDate(session.startTime)} {if (session.endTime > 0) formatDate(session.endTime)} {formatDuration(session.duration)} + {formatDuration(session.sessionCpuTime / 1000000)} {session.totalOperations} } @@ -484,6 +486,7 @@ private class StatementStatsPagedTable( ("Create Time", true, None), ("Finish Time", true, None), ("Duration", true, None), + ("Cpu Time", true, None), ("Statement", true, None), ("State", true, None), ("Query Details", true, None), @@ -523,6 +526,7 @@ private class StatementStatsPagedTable( {formatDuration(event.duration)} + {formatDuration(event.operationCpuTime.getOrElse(0L) / 1000000)} {event.statement} @@ -592,6 +596,7 @@ private class SessionStatsTableDataSource( case "Start Time" => Ordering.by(_.startTime) case "Finish Time" => Ordering.by(_.endTime) case "Duration" => Ordering.by(_.duration) + case "Cpu Time" => Ordering.by(_.sessionCpuTime) case "Total Statements" => Ordering.by(_.totalOperations) case unknownColumn => throw new IllegalArgumentException(s"Unknown column: $unknownColumn") } @@ -627,6 +632,7 @@ private class StatementStatsTableDataSource( case "Create Time" => Ordering.by(_.createTime) case "Finish Time" => Ordering.by(_.completeTime) case "Duration" => Ordering.by(_.duration) + case "Cpu Time" => Ordering.by(_.operationCpuTime.getOrElse(0L)) case "Statement" => Ordering.by(_.statement) case "State" => Ordering.by(_.state) case "Query Details" => Ordering.by(_.executionId) From 9018a07ede4af3b56527145faf4f97c61c6e6318 Mon Sep 17 00:00:00 2001 From: bkhan Date: Thu, 29 Feb 2024 11:36:32 +0800 Subject: [PATCH 2/4] Apply suggestions from code review Co-authored-by: Cheng Pan --- .../src/main/scala/org/apache/spark/ui/EnginePage.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EnginePage.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EnginePage.scala index aa34bc91239..67f8f30d34b 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EnginePage.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EnginePage.scala @@ -402,7 +402,7 @@ case class EnginePage(parent: EngineTab) extends WebUIPage("") { ("Start Time", true, None), ("Finish Time", true, None), ("Duration", true, None), - ("Cpu Time", true, None), + ("CPU Time", true, None), ("Total Statements", true, None)) headerStatRow( @@ -486,7 +486,7 @@ private class StatementStatsPagedTable( ("Create Time", true, None), ("Finish Time", true, None), ("Duration", true, None), - ("Cpu Time", true, None), + ("CPU Time", true, None), ("Statement", true, None), ("State", true, None), ("Query Details", true, None), @@ -596,7 +596,7 @@ private class SessionStatsTableDataSource( case "Start Time" => Ordering.by(_.startTime) case "Finish Time" => Ordering.by(_.endTime) case "Duration" => Ordering.by(_.duration) - case "Cpu Time" => Ordering.by(_.sessionCpuTime) + case "CPU Time" => Ordering.by(_.sessionCpuTime) case "Total Statements" => Ordering.by(_.totalOperations) case unknownColumn => throw new IllegalArgumentException(s"Unknown column: $unknownColumn") } @@ -632,7 +632,7 @@ private class StatementStatsTableDataSource( case "Create Time" => Ordering.by(_.createTime) case "Finish Time" => Ordering.by(_.completeTime) case "Duration" => Ordering.by(_.duration) - case "Cpu Time" => Ordering.by(_.operationCpuTime.getOrElse(0L)) + case "CPU Time" => Ordering.by(_.operationCpuTime.getOrElse(0L)) case "Statement" => Ordering.by(_.statement) case "State" => Ordering.by(_.state) case "Query Details" => Ordering.by(_.executionId) From 8e957c7ecd418a3bd43f3e3239aca6425a3bb513 Mon Sep 17 00:00:00 2001 From: bkhan Date: Mon, 4 Mar 2024 22:43:26 +0800 Subject: [PATCH 3/4] display run time --- .../src/main/scala/org/apache/spark/ui/EnginePage.scala | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EnginePage.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EnginePage.scala index 67f8f30d34b..d59b64dd9d2 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EnginePage.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EnginePage.scala @@ -402,6 +402,7 @@ case class EnginePage(parent: EngineTab) extends WebUIPage("") { ("Start Time", true, None), ("Finish Time", true, None), ("Duration", true, None), + ("Run Time", true, None), ("CPU Time", true, None), ("Total Statements", true, None)) @@ -429,6 +430,7 @@ case class EnginePage(parent: EngineTab) extends WebUIPage("") { {formatDate(session.startTime)} {if (session.endTime > 0) formatDate(session.endTime)} {formatDuration(session.duration)} + {formatDuration(session.sessionRunTime)} {formatDuration(session.sessionCpuTime / 1000000)} {session.totalOperations} @@ -486,6 +488,7 @@ private class StatementStatsPagedTable( ("Create Time", true, None), ("Finish Time", true, None), ("Duration", true, None), + ("Run Time", true, None), ("CPU Time", true, None), ("Statement", true, None), ("State", true, None), @@ -526,6 +529,7 @@ private class StatementStatsPagedTable( {formatDuration(event.duration)} + {formatDuration(event.operationRunTime.getOrElse(0L))} {formatDuration(event.operationCpuTime.getOrElse(0L) / 1000000)} @@ -596,6 +600,7 @@ private class SessionStatsTableDataSource( case "Start Time" => Ordering.by(_.startTime) case "Finish Time" => Ordering.by(_.endTime) case "Duration" => Ordering.by(_.duration) + case "Run Time" => Ordering.by(_.sessionRunTime) case "CPU Time" => Ordering.by(_.sessionCpuTime) case "Total Statements" => Ordering.by(_.totalOperations) case unknownColumn => throw new IllegalArgumentException(s"Unknown column: $unknownColumn") @@ -632,6 +637,7 @@ private class StatementStatsTableDataSource( case "Create Time" => Ordering.by(_.createTime) case "Finish Time" => Ordering.by(_.completeTime) case "Duration" => Ordering.by(_.duration) + case "Run Time" => Ordering.by(_.operationRunTime.getOrElse(0L)) case "CPU Time" => Ordering.by(_.operationCpuTime.getOrElse(0L)) case "Statement" => Ordering.by(_.statement) case "State" => Ordering.by(_.state) From aad2bbb6c434a64e9e696f17f4788eb5e698f98b Mon Sep 17 00:00:00 2001 From: bkhan Date: Tue, 5 Mar 2024 14:00:10 +0800 Subject: [PATCH 4/4] session immediately --- .../kyuubi/engine/spark/operation/SparkOperation.scala | 5 +++-- .../kyuubi/engine/spark/session/SparkSessionImpl.scala | 9 +++------ 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/SparkOperation.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/SparkOperation.scala index da65d9f644b..ec9db84e48d 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/SparkOperation.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/SparkOperation.scala @@ -135,8 +135,9 @@ abstract class SparkOperation(session: Session) info(s"statementId=${statementId}, " + s"operationRunTime=${formatDuration(l.getOperationRunTime)}, " + s"operationCpuTime=${formatDuration(l.getOperationCpuTime / 1000000)}") - session.asInstanceOf[SparkSessionImpl].increaseRunTime(l.getOperationRunTime) - session.asInstanceOf[SparkSessionImpl].increaseCpuTime(l.getOperationCpuTime) + session.asInstanceOf[SparkSessionImpl].increaseRunAndCpuTime( + l.getOperationRunTime, + l.getOperationCpuTime) }) } } diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/session/SparkSessionImpl.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/session/SparkSessionImpl.scala index e899c28c3ac..b0e2d91c53b 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/session/SparkSessionImpl.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/session/SparkSessionImpl.scala @@ -129,11 +129,8 @@ class SparkSessionImpl( handle) } - def increaseRunTime(time: Long): Unit = { - sessionRunTime.getAndAdd(time) - } - - def increaseCpuTime(time: Long): Unit = { - sessionCpuTime.getAndAdd(time) + def increaseRunAndCpuTime(runTime: Long, cpuTime: Long): Unit = { + sessionEvent.sessionRunTime = sessionRunTime.addAndGet(runTime) + sessionEvent.sessionCpuTime = sessionCpuTime.addAndGet(cpuTime) } }