Skip to content

Commit

Permalink
Remove TaskSpeedupFactor from QualificationSummaryInfo
Browse files Browse the repository at this point in the history
Signed-off-by: Ahmed Hussein <[email protected]>
  • Loading branch information
amahussein committed Nov 5, 2024
1 parent 51fb686 commit b698b2c
Showing 1 changed file with 1 addition and 9 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -181,12 +181,6 @@ class QualificationAppInfo(
all.map(_.unsupportedTaskDur).sum
}

private def calculateSpeedupFactor(all: Seq[StageQualSummaryInfo]): Double = {
val allSpeedupFactors = all.filter(_.stageTaskTime > 0).map(_.averageSpeedup)
val res = SQLPlanParser.averageSpeedup(allSpeedupFactors)
res
}

private def getAllReadFileFormats: Seq[String] = {
dataSourceInfo.map { ds =>
s"${ds.format.toLowerCase()}[${ds.schema}]"
Expand Down Expand Up @@ -575,7 +569,6 @@ class QualificationAppInfo(
// note that these ratios are based off the stage times which may be missing some stage
// overhead or execs that didn't have associated stages
val supportedSQLTaskDuration = calculateSQLSupportedTaskDuration(allStagesSummary)
val taskSpeedupFactor = calculateSpeedupFactor(allStagesSummary)
// Get all the unsupported Execs from the plan
val unSupportedExecs = planInfos.flatMap { p =>
// WholeStageCodeGen is excluded from the result.
Expand Down Expand Up @@ -635,7 +628,7 @@ class QualificationAppInfo(
notSupportFormatAndTypesString, getAllReadFileFormats, writeFormat,
allComplexTypes, nestedComplexTypes, longestSQLDuration, sqlDataframeTaskDuration,
nonSQLTaskDuration, unsupportedSQLTaskDuration, supportedSQLTaskDuration,
taskSpeedupFactor, info.sparkUser, info.startTime, estimatedInfo.sqlDfDuration,
info.sparkUser, info.startTime, estimatedInfo.sqlDfDuration,
origPlanInfos, origPlanInfosSummary.map(_.stageSum).flatten,
perSqlStageSummary.map(_.stageSum).flatten, estimatedInfo, perSqlInfos,
unSupportedExecs, unSupportedExprs, clusterTags, allClusterTagsMap,
Expand Down Expand Up @@ -961,7 +954,6 @@ case class QualificationSummaryInfo(
nonSqlTaskDurationAndOverhead: Long,
unsupportedSQLTaskDuration: Long,
supportedSQLTaskDuration: Long,
taskSpeedupFactor: Double,
user: String,
startTime: Long,
sparkSqlDFWallClockDuration: Long,
Expand Down

0 comments on commit b698b2c

Please sign in to comment.