Merge pull request #71 from rajdeepd/branch-ed2

fixed minor issues in Chapter 4 and 1
ml-resources · May 16, 2017 · 95f8470 · 95f8470 · zou-xuan · May 18, 2017
2 parents 59f73c0 + da7f298
commit 95f8470
Show file tree

Hide file tree

Showing 3 changed files with 13 additions and 5 deletions.
diff --git a/Chapter_01/python-spark-app/pythonapp.py b/Chapter_01/python-spark-app/pythonapp.py
@@ -1,10 +1,16 @@
 """A simple Spark app in Python"""
 import os
 import sys
+os.environ['SPARK_HOME']="/home/ubuntu/work/spark-2.0.0-bin-hadoop2.7/"
+SPARK_HOME = os.environ['SPARK_HOME']
+
+# Add the PySpark/py4j to the Python Path
+sys.path.insert(0, os.path.join(SPARK_HOME, "python", "lib"))
+sys.path.insert(0, os.path.join(SPARK_HOME, "python"))
 from pyspark import SparkContext
 from pyspark import SparkConf
 
-os.environ['SPARK_HOME']="/home/ubuntu/work/spark-2.0.0-bin-hadoop2.7/"
+
 
 # Append pyspark  to Python Path
 sys.path.append("/home/ubuntu/work/spark-2.0.0-bin-hadoop2.7")

diff --git a/Chapter_04/python/2.0.0/com/sparksamples/util.py b/Chapter_04/python/2.0.0/com/sparksamples/util.py
@@ -1,12 +1,13 @@
 import os
 import sys
-from pyspark.sql.types import *
+
 PATH = "/home/ubuntu/work/ml-resources/spark-ml/data"
 SPARK_HOME = "/home/ubuntu/work/spark-2.0.0-bin-hadoop2.7/"
 
 os.environ['SPARK_HOME'] = SPARK_HOME
 sys.path.append(SPARK_HOME + "/python")
 
+from pyspark.sql.types import *
 from pyspark import SparkContext
 from pyspark import SparkConf
 from pyspark.sql import SparkSession

diff --git a/Chapter_10/scala-2.0.x/build.sbt b/Chapter_10/scala-2.0.x/build.sbt
@@ -4,7 +4,8 @@ version := "1.0"
 
 scalaVersion := "2.11.7"
 
-libraryDependencies += "org.apache.spark" %% "spark-core" % "2.0.2"
-libraryDependencies += "org.apache.spark" %% "spark-mllib" % "2.0.2"
-
+libraryDependencies += "org.apache.spark" %% "spark-core" % "2.1.0"
+libraryDependencies += "org.apache.spark" %% "spark-mllib" % "2.1.0"
+libraryDependencies += "com.roundeights" %% "hasher" % "1.2.0"
+libraryDependencies += "org.mindrot" % "jbcrypt" % "0.3m"