Update

ging · Oct 8, 2019 · c9b0a8c · c9b0a8c
1 parent 2537c44
commit c9b0a8c
Show file tree

Hide file tree

Showing 6 changed files with 40 additions and 32 deletions.
diff --git a/README.md b/README.md
@@ -30,7 +30,7 @@ cd fiware-cosmos-orion-spark-connector-examples
 Next, [download](https://github.com/ging/fiware-cosmos-orion-spark-connector/releases/latest) the connector JAR from the connector repository and from the `fiware-cosmos-orion-spark-connector` run:
 
 ```
-mvn install:install-file -Dfile=$(PATH_DOWNLOAD)/orion.spark.connector-1.0.jar -DgroupId=org.fiware.cosmos -DartifactId=orion.spark.connector -Dversion=1.0 -Dpackaging=jar
+mvn install:install-file -Dfile=$(PATH_DOWNLOAD)/orion.spark.connector-1.1.0.jar -DgroupId=org.fiware.cosmos -DartifactId=orion.spark.connector -Dversion=1.1.0 -Dpackaging=jar
 ```
 
 where `PATH_DOWNLOAD` is the path where you downloaded the JAR.
@@ -340,13 +340,13 @@ curl -v localhost:1026/v2/subscriptions -s -S -H 'Content-Type: application/json
 	"condition": {
   	"attrs": [
     	"pressure",
-	"temperature"
+	    "temperature"
   	]
 	}
   },
   "notification": {
 	"http": {
-  	"url": <b>"http://taskmanager:9001/notify"</b>
+  	"url":"http://spark-master:9001"
 	},
 	"attrs": [
   	"temperature",
@@ -369,7 +369,7 @@ Let's build a JAR package of the example. In it, we need to include all the depe
 This can be done through the `maven package` command without the `add-dependencies-for-IDEA` profile checked.
 This will build a JAR file under `target/orion.spark.connector.examples-1.1.0.jar`.
 
-### Submitting the job (In progress)
+### Submitting the job 
 
 
 Let's submit the Example 3 code to the Spark cluster we have deployed. In order to do this you can use the spark-submit command provided by Spark.

diff --git a/dependency-reduced-pom.xml b/dependency-reduced-pom.xml
@@ -4,7 +4,7 @@
   <groupId>org.fiware.cosmos</groupId>
   <artifactId>orion.spark.connector.examples</artifactId>
   <name>Fiware Orion Spark Connector Examples</name>
-  <version>1.0</version>
+  <version>1.1.0</version>
   <url>https://www.fiware.org/</url>
   <build>
     <plugins>
@@ -135,13 +135,13 @@
         <dependency>
           <groupId>org.apache.spark</groupId>
           <artifactId>spark-core_2.11</artifactId>
-          <version>2.4.0</version>
+          <version>2.4.3</version>
           <scope>compile</scope>
         </dependency>
         <dependency>
           <groupId>org.apache.spark</groupId>
           <artifactId>spark-streaming_2.11</artifactId>
-          <version>2.4.0</version>
+          <version>2.4.3</version>
           <scope>compile</scope>
         </dependency>
         <dependency>
@@ -189,7 +189,7 @@
     <scala.binary.version>2.11</scala.binary.version>
     <play2.version>2.4.6</play2.version>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-    <spark.version>2.4.0</spark.version>
+    <spark.version>2.4.3</spark.version>
   </properties>
 </project>
 
diff --git a/doc/index.md b/doc/index.md
@@ -15,10 +15,10 @@ git clone https://github.com/ging/fiware-cosmos-orion-flink-connector-examples
 cd fiware-cosmos-orion-flink-connector-examples
 ```
 
-Next, [download](https://github.com/ging/fiware-cosmos-orion-flink-connector/releases/tag/1.0) the connector JAR from the connector repository and from the `fiware-cosmos-orion-flink-connector` run:
+Next, [download](https://github.com/ging/fiware-cosmos-orion-flink-connector/releases/latest) the connector JAR from the connector repository and from the `fiware-cosmos-orion-flink-connector` run:
 
 ```
-mvn install:install-file -Dfile=$(PATH_DOWNLOAD)/orion.flink.connector-1.0.jar -DgroupId=org.fiware.cosmos -DartifactId=orion.flink.connector -Dversion=1.0 -Dpackaging=jar
+mvn install:install-file -Dfile=$(PATH_DOWNLOAD)/orion.flink.connector-1.1.0.jar -DgroupId=org.fiware.cosmos -DartifactId=orion.flink.connector -Dversion=1.1.0 -Dpackaging=jar
 ```
 
 where `PATH_DOWNLOAD` is the path where you downloaded the JAR.
@@ -384,7 +384,7 @@ Since we are going to run the code from inside a Flink Task Manager container, w
 
 Let's build a JAR package of the example. In it, we need to include all the dependencies we have used, such as the connector, but exclude some of the dependencies provided by the environment (Flink, Scala...).
 This can be done through the `maven package` command without the `add-dependencies-for-IDEA` profile checked.
-This will build a JAR file under `target/orion.flink.connector.examples-1.0.jar`.
+This will build a JAR file under `target/orion.flink.connector.examples-1.1.0.jar`.
 
 ### Submitting the job
 

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -3,32 +3,40 @@ services:
  mongo:
    image: mongo:3.4
    command: --nojournal
+   networks:
+     - fiware
  orion:
    image: fiware/orion
    links:
      - mongo
    ports:
      - "1026:1026"
    command: -dbhost mongo
- jobmanager:
-   image: flink:1.5.3-hadoop24-scala_2.11
-   expose:
-     - "6123"
+   networks:
+     - fiware
+ spark-master:
+   image: bde2020/spark-master:2.4.4-hadoop2.7
+   container_name: spark-master
    ports:
-     - "8081:8081"
-   command: jobmanager
+     - "8080:8080"
+     - "7077:7077"
+     - "9001:9001"
    environment:
-     - JOB_MANAGER_RPC_ADDRESS=jobmanager
-
- taskmanager:
-   image: flink:1.5.3-hadoop24-scala_2.11
-   expose:
-     - "6121"
-     - "6122"
+     - INIT_DAEMON_STEP=setup_spark
+     - "constraint:node==spark-master"
+   networks:
+     - fiware
+ spark-worker-1:
+   image: bde2020/spark-worker:2.4.4-hadoop2.7
+   container_name: spark-worker-1
    depends_on:
-     - jobmanager
-   command: taskmanager
-   links:
-     - "jobmanager:jobmanager"
+    - spark-master
+   ports:
+     - "8081:8081"
    environment:
-     - JOB_MANAGER_RPC_ADDRESS=jobmanager
+     - "SPARK_MASTER=spark://spark-master:7077"
+     - "constraint:node==spark-master"
+   networks:
+     - fiware
+networks:
+  fiware:
diff --git a/pom.xml b/pom.xml
@@ -23,7 +23,7 @@ under the License.
 
 	<groupId>org.fiware.cosmos</groupId>
 	<artifactId>orion.spark.connector.examples</artifactId>
-	<version>1.0.1</version>
+	<version>1.1.0</version>
 	<packaging>jar</packaging>
 
 	<name>Fiware Orion Spark Connector Examples</name>
@@ -87,7 +87,7 @@ under the License.
 		<dependency>
 			<groupId>org.fiware.cosmos</groupId>
 			<artifactId>orion.spark.connector</artifactId>
-			<version>1.0.4</version>
+			<version>1.1.0</version>
 		</dependency>
 
 		<!-- https://mvnrepository.com/artifact/org.json4s/json4s-jackson -->

diff --git a/src/main/scala/org/fiware/cosmos/orion/spark/connector/examples/example1/Example1.scala b/src/main/scala/org/fiware/cosmos/orion/spark/connector/examples/example1/Example1.scala
@@ -32,4 +32,4 @@ object Example1{
     ssc.start()
     ssc.awaitTermination()
   }
-}
+}
-Original file line number
+Diff line change
@@ Expand Up / @@ -32,4 +32,4 @@ object Example1{ @@
         ssc.start()
         ssc.awaitTermination()
       }
-    }
+    }