From e195e248e3bc7e895136b1281dd45fe070fe5f0a Mon Sep 17 00:00:00 2001 From: MrPowers Date: Thu, 1 Apr 2021 17:11:13 -0300 Subject: [PATCH 1/5] Switch dependencies to SBT --- build.sbt | 18 ++++++------------ project/build.properties | 2 +- project/plugins.sbt | 6 +----- 3 files changed, 8 insertions(+), 18 deletions(-) diff --git a/build.sbt b/build.sbt index d4a052b..6dec41f 100644 --- a/build.sbt +++ b/build.sbt @@ -4,16 +4,10 @@ version := "0.1" organization := "com.github.sramirez" -scalaVersion := "2.11.8" - -libraryDependencies += "org.apache.spark" %% "spark-mllib" % "2.2.0" - -resolvers ++= Seq( - "Apache Staging" at "https://repository.apache.org/content/repositories/staging/", - "Typesafe" at "http://repo.typesafe.com/typesafe/releases", - Resolver.mavenLocal -) - -publishMavenStyle := true - +scalaVersion := "2.11.12" +libraryDependencies += "org.apache.spark" %% "spark-core" % "2.2.0" % "provided" +libraryDependencies += "org.apache.spark" %% "spark-mllib" % "2.2.0" % "provided" +libraryDependencies += "org.scalatest" %% "scalatest" % "2.2.6" % "test" +libraryDependencies += "joda-time" % "joda-time" % "2.9.4" % "test" +libraryDependencies += "junit" % "junit" % "4.12" % "test" diff --git a/project/build.properties b/project/build.properties index 4b52bb9..622839d 100644 --- a/project/build.properties +++ b/project/build.properties @@ -14,4 +14,4 @@ # See the License for the specific language governing permissions and # limitations under the License. # -sbt.version=0.13.1 +sbt.version=1.4.3 diff --git a/project/plugins.sbt b/project/plugins.sbt index 932a99f..9ebee6c 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,5 +1 @@ -addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.12.0") - -resolvers += "Spark Package Main Repo" at "https://dl.bintray.com/spark-packages/maven" - -addSbtPlugin("org.spark-packages" % "sbt-spark-package" % "0.2.1") +addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.4.2") \ No newline at end of file From d3efa245c3e958b59d4f981adeba8efb8887b2a1 Mon Sep 17 00:00:00 2001 From: MrPowers Date: Thu, 1 Apr 2021 17:15:28 -0300 Subject: [PATCH 2/5] Delete pom file --- pom.xml | 106 -------------------------------------------------------- 1 file changed, 106 deletions(-) delete mode 100644 pom.xml diff --git a/pom.xml b/pom.xml deleted file mode 100644 index c068fbb..0000000 --- a/pom.xml +++ /dev/null @@ -1,106 +0,0 @@ - - 4.0.0 - 1.4.4 - ${project.artifactId} - sramirez - spark-infotheoretic-feature-selection - 2016 - - 1.6 - 1.6 - UTF-8 - 2.11 - 2.11.6 - - - - org.apache.spark - spark-core_2.11 - 2.2.0 - - - org.apache.spark - spark-mllib_2.11 - 2.2.0 - - - org.scala-lang - scala-library - ${scala.version} - - - - org.scalatest - scalatest_${scala.tools.version} - 2.2.6 - test - - - junit - junit - 4.12 - test - - - joda-time - joda-time - 2.9.4 - - - - - src/main/scala - src/test/scala - - - - net.alchim31.maven - scala-maven-plugin - 3.1.3 - - - - compile - testCompile - - - - - -dependencyfile - ${project.build.directory}/.scala_dependencies - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - 2.13 - - false - true - - - - **/*Test.* - **/*Suite.* - - - - - - org.apache.maven.plugins - maven-source-plugin - - - attach-sources - - jar - - - - - - - From 1fd10c1cb8c6e6c47a17d1658915c7a2d2d1ffd1 Mon Sep 17 00:00:00 2001 From: MrPowers Date: Thu, 1 Apr 2021 17:27:49 -0300 Subject: [PATCH 3/5] Bump to Spark 3 --- build.sbt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build.sbt b/build.sbt index 6dec41f..3db397c 100644 --- a/build.sbt +++ b/build.sbt @@ -4,10 +4,10 @@ version := "0.1" organization := "com.github.sramirez" -scalaVersion := "2.11.12" +scalaVersion := "2.12.12" -libraryDependencies += "org.apache.spark" %% "spark-core" % "2.2.0" % "provided" -libraryDependencies += "org.apache.spark" %% "spark-mllib" % "2.2.0" % "provided" -libraryDependencies += "org.scalatest" %% "scalatest" % "2.2.6" % "test" +libraryDependencies += "org.apache.spark" %% "spark-core" % "3.0.1" % "provided" +libraryDependencies += "org.apache.spark" %% "spark-mllib" % "3.0.1" % "provided" +libraryDependencies += "org.scalatest" %% "scalatest" % "3.0.1" % "test" libraryDependencies += "joda-time" % "joda-time" % "2.9.4" % "test" libraryDependencies += "junit" % "junit" % "4.12" % "test" From a4a9ea0551a21f43391fbbc4f08b46ee1227102d Mon Sep 17 00:00:00 2001 From: MrPowers Date: Fri, 2 Apr 2021 12:35:55 -0300 Subject: [PATCH 4/5] Comment out broken code --- src/main/scala/org/apache/spark/ml/feature/InfoThSelector.scala | 2 +- .../scala/org/apache/spark/mllib/feature/InfoThSelector.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/scala/org/apache/spark/ml/feature/InfoThSelector.scala b/src/main/scala/org/apache/spark/ml/feature/InfoThSelector.scala index d55cfec..fe69431 100644 --- a/src/main/scala/org/apache/spark/ml/feature/InfoThSelector.scala +++ b/src/main/scala/org/apache/spark/ml/feature/InfoThSelector.scala @@ -257,7 +257,7 @@ object InfoThSelectorModel extends MLReadable[InfoThSelectorModel] { val selectedFeatures = data.getAs[Seq[Int]](0).toArray val oldModel = new feature.InfoThSelectorModel(selectedFeatures) val model = new InfoThSelectorModel(metadata.uid, oldModel) - DefaultParamsReader.getAndSetParams(model, metadata) +// DefaultParamsReader.getAndSetParams(model, metadata) model } } diff --git a/src/main/scala/org/apache/spark/mllib/feature/InfoThSelector.scala b/src/main/scala/org/apache/spark/mllib/feature/InfoThSelector.scala index b2b1a48..936845a 100644 --- a/src/main/scala/org/apache/spark/mllib/feature/InfoThSelector.scala +++ b/src/main/scala/org/apache/spark/mllib/feature/InfoThSelector.scala @@ -124,7 +124,7 @@ class InfoThSelectorModel @Since("1.6.0") ( InfoThSelectorModel.SaveLoadV1_0.save(sc, this, path) } - override protected def formatVersion: String = "1.0" +// override protected def formatVersion: String = "1.0" } object InfoThSelectorModel extends Loader[InfoThSelectorModel] { From 2a982f7a1b629c9212a7b3b893f643706561eb3a Mon Sep 17 00:00:00 2001 From: Simeon Simeonov Date: Wed, 28 Apr 2021 20:39:26 -0400 Subject: [PATCH 5/5] Refactors getAndSetParams use --- src/main/scala/org/apache/spark/ml/feature/InfoThSelector.scala | 2 +- .../scala/org/apache/spark/mllib/feature/InfoThSelector.scala | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/scala/org/apache/spark/ml/feature/InfoThSelector.scala b/src/main/scala/org/apache/spark/ml/feature/InfoThSelector.scala index fe69431..1f0dabf 100644 --- a/src/main/scala/org/apache/spark/ml/feature/InfoThSelector.scala +++ b/src/main/scala/org/apache/spark/ml/feature/InfoThSelector.scala @@ -257,7 +257,7 @@ object InfoThSelectorModel extends MLReadable[InfoThSelectorModel] { val selectedFeatures = data.getAs[Seq[Int]](0).toArray val oldModel = new feature.InfoThSelectorModel(selectedFeatures) val model = new InfoThSelectorModel(metadata.uid, oldModel) -// DefaultParamsReader.getAndSetParams(model, metadata) + metadata.getAndSetParams(model) model } } diff --git a/src/main/scala/org/apache/spark/mllib/feature/InfoThSelector.scala b/src/main/scala/org/apache/spark/mllib/feature/InfoThSelector.scala index 936845a..f27f2ff 100644 --- a/src/main/scala/org/apache/spark/mllib/feature/InfoThSelector.scala +++ b/src/main/scala/org/apache/spark/mllib/feature/InfoThSelector.scala @@ -124,7 +124,6 @@ class InfoThSelectorModel @Since("1.6.0") ( InfoThSelectorModel.SaveLoadV1_0.save(sc, this, path) } -// override protected def formatVersion: String = "1.0" } object InfoThSelectorModel extends Loader[InfoThSelectorModel] {