Merge pull request #10 from benmccann/cleanup

Minor code cleanup
zhengruifeng · Aug 13, 2018 · fc598a1 · fc598a1
2 parents 8247fb6 + f9beb86
commit fc598a1
Show file tree

Hide file tree

Showing 4 changed files with 6 additions and 27 deletions.
diff --git a/pom.xml b/pom.xml
@@ -18,26 +18,6 @@
     </properties>
 
     <dependencies>
-        <dependency>
-            <groupId>org.apache.spark</groupId>
-            <artifactId>spark-core_2.10</artifactId>
-            <version>${spark.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.spark</groupId>
-            <artifactId>spark-sql_2.10</artifactId>
-            <version>${spark.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.spark</groupId>
-            <artifactId>spark-hive_2.10</artifactId>
-            <version>${spark.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.spark</groupId>
-            <artifactId>spark-graphx_2.10</artifactId>
-            <version>${spark.version}</version>
-        </dependency>
         <dependency>
             <groupId>org.apache.spark</groupId>
             <artifactId>spark-mllib_2.10</artifactId>
@@ -63,7 +43,6 @@
                 <artifactId>scala-maven-plugin</artifactId>
                 <version>3.1.6</version>
                 <executions>
-
                     <execution>
                         <id>compile</id>
                         <goals>

diff --git a/src/main/scala/org/apache/spark/mllib/regression/FMWithLBFGS.scala b/src/main/scala/org/apache/spark/mllib/regression/FMWithLBFGS.scala
@@ -139,7 +139,7 @@ class FMWithLBFGS(private var task: Int,
 
   /**
    * Encode the FMModel to a dense vector, with its first numFeatures * numFactors elements representing the
-   * factorization matrix v, sequential numFeaturs elements representing the one-way interactions weights w if k1 is
+   * factorization matrix v, sequential numFeatures elements representing the one-way interactions weights w if k1 is
    * set to true, and the last element representing the intercept w0 if k0 is set to true.
    * The factorization matrix v is initialized by Gaussinan(0, initStd).
    * v : numFeatures * numFactors + w : [numFeatures] + w0 : [1]
@@ -165,7 +165,7 @@ class FMWithLBFGS(private var task: Int,
 
 
   /**
-   * Create a FMModle from an encoded vector.
+   * Create a FMModel from an encoded vector.
    */
   private def createModel(weights: Vector): FMModel = {
 

diff --git a/src/main/scala/org/apache/spark/mllib/regression/FMWithSGD.scala b/src/main/scala/org/apache/spark/mllib/regression/FMWithSGD.scala
@@ -169,7 +169,7 @@ class FMWithSGD(private var task: Int,
 
   /**
     * Encode the FMModel to a dense vector, with its first numFeatures * numFactors elements representing the
-    * factorization matrix v, sequential numFeaturs elements representing the one-way interactions weights w if k1 is
+    * factorization matrix v, sequential numFeatures elements representing the one-way interactions weights w if k1 is
     * set to true, and the last element representing the intercept w0 if k0 is set to true.
     * The factorization matrix v is initialized by Gaussinan(0, initStd).
     * v : numFeatures * numFactors + w : [numFeatures] + w0 : [1]
@@ -195,7 +195,7 @@ class FMWithSGD(private var task: Int,
 
 
   /**
-    * Create a FMModle from an encoded vector.
+    * Create a FMModel from an encoded vector.
     */
   private def createModel(weights: Vector): FMModel = {
 

diff --git a/src/main/scala/org/apache/spark/mllib/regression/FactorizationMachine.scala b/src/main/scala/org/apache/spark/mllib/regression/FactorizationMachine.scala
@@ -107,13 +107,13 @@ object FMModel extends Loader[FMModel] {
 
       // Create Parquet data.
       val dataRDD: DataFrame = sc.parallelize(Seq(data), 1).toDF()
-      dataRDD.saveAsParquetFile(dataPath(path))
+      dataRDD.write.parquet(dataPath(path))
     }
 
     def load(sc: SparkContext, path: String): FMModel = {
       val sqlContext = new SQLContext(sc)
       // Load Parquet data.
-      val dataRDD = sqlContext.parquetFile(dataPath(path))
+      val dataRDD = sqlContext.read.parquet(dataPath(path))
       // Check schema explicitly since erasure makes it hard to use match-case for checking.
       checkSchema[Data](dataRDD.schema)
       val dataArray = dataRDD.select("task", "factorMatrix", "weightVector", "intercept", "min", "max").take(1)