Add Reflection to support custom Spark Implementation at Runtime (#1362)

* Add Reflection to support custom Spark Implementation at Runtime

Signed-off-by: Ahmed Hussein <[email protected]>

Fixes #1360

Adds a workaorund to run against open source Spark and custom Spark
implementation that overrides the constructors of Graph objects.

* Address the PR comments

Signed-off-by: Ahmed Hussein <[email protected]>

* Remove extra unecessary comment

Signed-off-by: Ahmed Hussein <[email protected]>

* Fix typo

Signed-off-by: Ahmed Hussein <[email protected]>

---------

Signed-off-by: Ahmed Hussein <[email protected]>

core/src/main/scala/com/nvidia/spark/rapids/SparkRapidsBuildInfoEvent.scala

@@ -17,6 +17,7 @@
 package com.nvidia.spark.rapids
 
 import org.apache.spark.scheduler.SparkListenerEvent
+import org.apache.spark.sql.rapids.tool.annotation.ToolsReflection
 
 
 /**
@@ -30,4 +31,9 @@ case class SparkRapidsBuildInfoEvent(
   sparkRapidsJniBuildInfo: Map[String, String],
   cudfBuildInfo: Map[String, String],
   sparkRapidsPrivateBuildInfo: Map[String, String]
-) extends SparkListenerEvent
+) extends SparkListenerEvent {
+  @ToolsReflection("BD-3.2.1", "Ignore")
+  override val eventTime: Long = 0
+  @ToolsReflection("BD-3.2.1", "Ignore")
+  override val eventType: String = ""
+}

core/src/main/scala/org/apache/spark/scheduler/SparkListenerEvent.scala

@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2024, NVIDIA CORPORATION.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.scheduler
+
+import com.fasterxml.jackson.annotation.JsonTypeInfo
+
+import org.apache.spark.annotation.DeveloperApi
+
+/**
+ * This code is mostly copied from org.apache.spark.scheduler.SparkListenerEvent
+ * to make it compatible at runtime with custom Spark implementation that defines abstract methods
+ * in the trait.
+ *
+ * This class is packaged due to a bug in Scala 2.12 that links the method
+ * to the abstract trait, which might not exist in the classpath.
+ * See the related Scala issues:
+ *   https://github.com/scala/bug/issues/10477
+ *   https://github.com/scala/scala-dev/issues/219
+ *   https://github.com/scala/scala-dev/issues/268
+ */
+@DeveloperApi
+@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, include = JsonTypeInfo.As.PROPERTY, property = "Event")
+trait SparkListenerEvent extends ToolsListenerEventExtraAPIs {
+  /* Whether output this event to the event log */
+  protected[spark] def logEvent: Boolean = true
+}

core/src/main/scala/org/apache/spark/scheduler/ToolsListenerEventExtraAPIs.scala

@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2024, NVIDIA CORPORATION.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.scheduler
+
+import org.apache.spark.sql.rapids.tool.annotation.ToolsReflection
+
+/**
+ * Base trait for events related to SparkRapids build info. This used to add extra APIs that are
+ * not defined in the base Spark trait. This is a work around to be compatible in
+ * runtime with custom Spark implementations that define abstract methods in the trait.
+ * see https://github.com/NVIDIA/spark-rapids-tools/issues/1360
+ */
+trait ToolsListenerEventExtraAPIs {
+  @ToolsReflection("BD-3.2.1",
+    "Ignore the implementation: Definition for an abstract field in the SparkListenerEvent.")
+  val eventTime: Long = 0
+  @ToolsReflection("BD-3.2.1",
+    "Ignore the implementation: Definition for an abstract field in the SparkListenerEvent.")
+  val eventType: String = ""
+}

core/src/main/scala/org/apache/spark/sql/rapids/tool/annotation/ToolsReflection.scala

@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2024, NVIDIA CORPORATION.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.rapids.tool.annotation
+
+import scala.annotation.StaticAnnotation
+import scala.annotation.meta.{beanGetter, beanSetter, field, getter, param, setter}
+
+
+/**
+ * This code is mostly copied from org.apache.spark.annotation.Since
+ * Reason is copied here because it is being private to Spark packages which makes it
+ * inaccessible for Non-Spark packages.
+ *
+ * A Scala annotation that indicates entities that are used for reeflection in Tools to match
+ * different Spark runtime APIs
+ */
+@param @field @getter @setter @beanGetter @beanSetter
+class ToolsReflection(source: String, comment: String) extends StaticAnnotation