Support for Spark 2.4.3

.travis.yml

@@ -8,12 +8,12 @@ cache:
 matrix:
   include: 
     - jdk: oraclejdk8 
-      scala: 2.11.8
+      scala: 2.11.12
       env: TEST_HADOOP_VERSION="2.7.3" TEST_SPARK_VERSION="2.3.1"
     - jdk: oraclejdk8
-      scala: 2.11.8
-      env: TEST_HADOOP_VERSION="2.2.0" TEST_SPARK_VERSION="2.1.0"
+      scala: 2.11.12
+      env: TEST_HADOOP_VERSION="2.7.7" TEST_SPARK_VERSION="2.4.3"
 script:
-  - sbt -Dhadoop.testVersion=$TEST_HADOOP_VERSION -Dspark.testVersion=$TEST_SPARK_VERSION ++$TRAVIS_SCALA_VERSION coverage test
+  - sbt -Dhadoop.testVersion=$TEST_HADOOP_VERSION -Dspark.testVersion=$TEST_SPARK_VERSION ++$TRAVIS_SCALA_VERSION clean coverage test coverageReport
 after_success:
   - bash <(curl -s https://codecov.io/bash)

build.sbt

@@ -4,11 +4,11 @@ version := "1.0.6-SNAPSHOT"
 
 organization := "harsha2010"
 
-scalaVersion := "2.11.8"
+scalaVersion := "2.11.12"
 
-crossScalaVersions := Seq("2.11.8")
+crossScalaVersions := Seq("2.11.12")
 
-sparkVersion := "2.3.1"
+sparkVersion := "2.4.3"
 
 scalacOptions += "-optimize"
 
@@ -18,20 +18,30 @@ testSparkVersion := sys.props.get("spark.testVersion").getOrElse(sparkVersion.va
 
 val testHadoopVersion = settingKey[String]("The version of Hadoop to test against.")
 
-testHadoopVersion := sys.props.getOrElse("hadoop.testVersion", "2.7.3")
+testHadoopVersion := sys.props.getOrElse("hadoop.testVersion", "2.7.7")
 
 sparkComponents := Seq("core", "sql")
 
 libraryDependencies ++= Seq(
-  "commons-io" % "commons-io" % "2.4",
-  "com.google.guava" % "guava" % "14.0.1" % "provided",
+  "commons-io" % "commons-io" % "2.6",
+  "com.google.guava" % "guava" % "27.1-jre",
   "org.slf4j" % "slf4j-api" % "1.7.16" % "provided",
-  "com.lihaoyi" % "fastparse_2.11" % "0.4.3" % "provided",
-  "org.scalatest" %% "scalatest" % "2.2.1" % "test",
+  "com.lihaoyi" %% "fastparse" % "0.4.4" % "provided",
+  "org.scalatest" %% "scalatest" % "3.0.5" % "test",
   "com.vividsolutions" % "jts" % "1.13" % "test",
-  "com.esri.geometry" % "esri-geometry-api" % "1.2.1"
+  "com.esri.geometry" % "esri-geometry-api" % "1.2.1",
+  "com.fasterxml.jackson.core" % "jackson-core" % "2.9.8",
+  "com.fasterxml.jackson.core" % "jackson-databind" % "2.9.8",
+  "com.fasterxml.jackson.core" % "jackson-annotations" % "2.9.8",
+  "com.fasterxml.jackson.jaxrs" % "jackson-jaxrs-base" % "2.9.8",
+  "com.fasterxml.jackson.jaxrs" % "jackson-jaxrs-json-provider" % "2.9.8",
+  "com.fasterxml.jackson.module" % "jackson-module-paranamer" % "2.9.8",
+  "com.fasterxml.jackson.module" %% "jackson-module-scala" % "2.9.8"
+
 )
 
+libraryDependencies += "org.scala-lang" % "scala-compiler" % scalaVersion.value % "provided"
+
 libraryDependencies ++= Seq(
   "org.apache.hadoop" % "hadoop-client" % testHadoopVersion.value % "test",
   "org.apache.spark" %% "spark-core" % testSparkVersion.value % "test" exclude("org.apache.hadoop", "hadoop-client"),
@@ -81,7 +91,8 @@ spName := "harsha2010/magellan"
 
 parallelExecution in Test := false
 
-ScoverageSbtPlugin.ScoverageKeys.coverageHighlighting := {
+
+scoverage.ScoverageKeys.coverageHighlighting := {
   if (scalaBinaryVersion.value == "2.10") false
   else true
 }

project/assembly.sbt

@@ -1 +1 @@
-addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.13.0")
+addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.9")

project/build.properties

@@ -13,4 +13,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-sbt.version=0.13.6
+sbt.version=0.13.18

project/plugins.sbt

@@ -1,7 +1,5 @@
 resolvers += "Spark Package Main Repo" at "https://dl.bintray.com/spark-packages/maven"
 
-addSbtPlugin("com.typesafe.sbteclipse" % "sbteclipse-plugin" % "2.2.0")
-
 addSbtPlugin("com.github.mpeltonen" % "sbt-idea" % "1.6.0")
 
 addSbtPlugin("org.spark-packages" % "sbt-spark-package" % "0.2.2")
@@ -19,6 +17,6 @@ addSbtPlugin("com.eed3si9n" % "sbt-unidoc" % "0.3.3")
 
 addSbtPlugin("com.jsuereth" % "sbt-pgp" % "1.0.0")
 
-addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.1.0")
+addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.5.1")
 
 addSbtPlugin("com.typesafe" % "sbt-mima-plugin" % "0.1.7")

src/main/scala/magellan/catalyst/SpatialJoin.scala

@@ -114,19 +114,19 @@ private[magellan] case class SpatialJoin(session: SparkSession)
     // isGenerated: java.lang.Boolean = false
     val klass = Class.forName("org.apache.spark.sql.catalyst.expressions.AttributeReference")
     val ctor = klass.getConstructors.apply(0)
-    val nullable = true.asInstanceOf[AnyRef]
+    val nullable = true
     val metadata = Metadata.empty
     val exprId = NamedExpression.newExprId
-    val qualifier = None
+    val qualifier = Seq.empty[String]
     val isGenerated = false.asInstanceOf[AnyRef]
     if (ctor.getParameterCount == 7) {
       // prior to Spark 2.3
-      ctor.newInstance(name, dt, nullable, metadata, exprId, qualifier, isGenerated)
+      ctor.newInstance(name, dt, nullable.asInstanceOf[AnyRef], metadata, exprId, qualifier, isGenerated)
         .asInstanceOf[Attribute]
     } else {
       // Spark 2.3  +
-      ctor.newInstance(name, dt, nullable, metadata, exprId, qualifier)
-        .asInstanceOf[Attribute]
+      AttributeReference(name, dt, nullable, metadata)(exprId, qualifier).asInstanceOf[Attribute]
+
     }
 
   }

src/main/scala/magellan/mapreduce/ShapeInputFormat.scala

@@ -47,7 +47,7 @@ private[magellan] class ShapeInputFormat
        job: JobContext,
        splitInfos: scala.collection.Map[String, Array[Long]]) = {
 
-    val sw = new Stopwatch().start
+    val sw = Stopwatch.createUnstarted().start()
     val splits = ListBuffer[InputSplit]()
     val files = listStatus(job)
     for (file <- files) {
@@ -77,7 +77,7 @@ private[magellan] class ShapeInputFormat
     }
     sw.stop
     if (log.isDebugEnabled) {
-      log.debug("Total # of splits generated by getSplits: " + splits.size + ", TimeTaken: " + sw.elapsedMillis)
+      log.debug("Total # of splits generated by getSplits: " + splits.size + ", TimeTaken: " + sw.elapsed(java.util.concurrent.TimeUnit.MILLISECONDS))
     }
     splits
   }