[paypal#247] [spark] Bump Spark Version to 2.4.7 | Bump Spark Version…

… to 2.4.7 along with other stack to align with gcp_dataproc_1.5.x
Dee-Pac · Nov 8, 2020 · 3b25dcb · 3b25dcb
1 parent 219fc00
commit 3b25dcb
Show file tree

Hide file tree

Showing 9 changed files with 4 additions and 4 deletions.
diff --git a/...l-connectors/gimel-aerospike-3.14/pom.xml → .../gimel-connectors/gimel-aerospike/pom.xml b/...l-connectors/gimel-aerospike-3.14/pom.xml → .../gimel-connectors/gimel-aerospike/pom.xml
@@ -28,7 +28,7 @@ under the License.
   </parent>
   <modelVersion>4.0.0</modelVersion>
 
-  <artifactId>gimel-aerospike-3.14</artifactId>
+  <artifactId>gimel-aerospike</artifactId>
   <version>2.4.7-SNAPSHOT</version>
 
   <dependencies>
@@ -58,7 +58,7 @@ under the License.
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-shade-plugin</artifactId>
-        <version>3.0.0</version>
+        <version>${maven.shade.plugin.version}</version>
         <configuration>
           <relocations>
             <relocation>

diff --git a/.../com/paypal/gimel/aerospike/DataSet.scala → .../com/paypal/gimel/aerospike/DataSet.scala b/.../com/paypal/gimel/aerospike/DataSet.scala → .../com/paypal/gimel/aerospike/DataSet.scala
diff --git a/...e/conf/AerospikeClientConfiguration.scala → ...e/conf/AerospikeClientConfiguration.scala b/...e/conf/AerospikeClientConfiguration.scala → ...e/conf/AerospikeClientConfiguration.scala
diff --git a/...mel/aerospike/conf/AerospikeConfigs.scala → ...mel/aerospike/conf/AerospikeConfigs.scala b/...mel/aerospike/conf/AerospikeConfigs.scala → ...mel/aerospike/conf/AerospikeConfigs.scala
diff --git a/...el/aerospike/reader/AerospikeReader.scala → ...el/aerospike/reader/AerospikeReader.scala b/...el/aerospike/reader/AerospikeReader.scala → ...el/aerospike/reader/AerospikeReader.scala
diff --git a/...ospike/utilities/AeroSparkConnector.scala → ...ospike/utilities/AeroSparkConnector.scala b/...ospike/utilities/AeroSparkConnector.scala → ...ospike/utilities/AeroSparkConnector.scala
diff --git a/...ospike/utilities/AerospikeUtilities.scala → ...ospike/utilities/AerospikeUtilities.scala b/...ospike/utilities/AerospikeUtilities.scala → ...ospike/utilities/AerospikeUtilities.scala
@@ -57,7 +57,7 @@ object AerospikeUtilities {
       val aerospikeSet = conf.aerospikeSet
       val aerospikeRowKey = conf.aerospikeRowKey
       // For each partition of Dataframe, aerospike client is created and is used to write data to aerospike
-      dataFrame.foreachPartition { partition =>
+      dataFrame.foreachPartition { partition: Iterator[Row] =>
         val client = AerospikeAdminClient.createClientConnection(aerospikeHosts, aerospikePort.toInt)
         partition.foreach { row =>
           val bins = columns.map(eachCol => new Bin(eachCol.toString, row.getAs(eachCol).toString)).toArray

diff --git a/...el/aerospike/writer/AerospikeWriter.scala → ...el/aerospike/writer/AerospikeWriter.scala b/...el/aerospike/writer/AerospikeWriter.scala → ...el/aerospike/writer/AerospikeWriter.scala
diff --git a/gimel-dataapi/pom.xml b/gimel-dataapi/pom.xml
@@ -47,7 +47,7 @@ under the License.
         <module>gimel-connectors/gimel-jdbc</module>
         <module>gimel-connectors/gimel-cassandra</module>
         <module>gimel-connectors/gimel-hbase</module>
-        <module>gimel-connectors/gimel-aerospike-3.14</module>
+        <module>gimel-connectors/gimel-aerospike</module>
         <module>gimel-connectors/gimel-druid-0.82</module>
         <module>gimel-core</module>
         <module>gimel-sql</module>