[paypal#247] [spark] Bump Spark Version to 2.4.7 | Bump Spark Version…

… to 2.4.7 along with other stack to align with gcp_dataproc_1.5.x
Dee-Pac · Nov 9, 2020 · f0f0049 · f0f0049
1 parent d78326b
commit f0f0049
Show file tree

Hide file tree

Showing 118 changed files with 374 additions and 635 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -29,4 +29,4 @@ before_install:
 
 # Build
 install: 
-- build/gimel -Dmaven.test.skip=true -pl gimel-dataapi/gimel-tools -am | egrep -v "Download|Copy|Including|WARNING"
+- sh -x build/gimel -Dmaven.test.skip=true -pl gimel-dataapi/gimel-tools -am | egrep -v "Download|Copy|Including|WARNING"
diff --git a/build/gimel b/build/gimel
@@ -15,9 +15,81 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+echo "Present Working Dir"
+echo $PWD
+
 this_dir=`dirname build/install_dependencies`
-# Source reusable functions
-source ${this_dir}/gimel_functions
+
+#----------------------------function will check for error code & exit if failure, else proceed further----------------------------#
+
+#usage : check_error <$?> <custom_error_code>
+#Example: Check_error < pass $? from the shell command > < Custom Message for errorcode -gt 0 >
+
+check_error()
+{
+	cmd_error_code=$1
+	custom_message=$2
+	if  [ ${cmd_error_code} -gt 0 ]; then
+	  write_log "Error    | Stage |  ${custom_message}"
+	  exit ${cmd_error_code}
+	else
+	  write_log "Success  | Stage | ${custom_message}"
+	fi
+}
+
+#----------------------------function will check for error code & warn if failure----------------------------#
+
+#usage : check_warning <$?> <custom_error_code>
+#Example: Check_warning < pass $? from the shell command > < Custom Message for errorcode -gt 0 >
+
+
+check_warning()
+{
+
+        cmd_error_code=$1
+        pgm_exit_code=$2
+        pgm_exit_msg=$3
+        if  [ ${cmd_error_code} -gt 0 ]; then
+                write_log "WARNING ! ${cmd_error_code} ${pgm_exit_code} ${pgm_exit_msg}"
+        else
+                echo ""
+        fi
+}
+
+
+
+#----------------------------function will write the message to Console / Log File----------------------------#
+
+#Usage : write_log < Whatever message you need to log >
+
+write_log()
+{
+        msg=$1
+        to_be_logged="$(date '+%Y%m%d %H:%M:%S') | $msg"
+        echo ${to_be_logged}
+}
+
+#-----------------------------------Executes a Command--------------------------------------------------------#
+
+
+
+#Usage : run_cmd < The command to execute > 
+
+run_cmd()
+{
+       cmd=$1
+       if [ -z $2 ]; then 
+         fail_on_error="break_code"
+       else
+         fail_on_error=$2
+       fi 
+       write_log "Executing Command --> $1"
+       $cmd
+       error_code=$?
+       if [ ! $fail_on_error = "ignore_errors" ]; then
+           check_error $error_code "$cmd"
+       fi
+}
 
 #--------------------------------Begin execution of Steps------------------------------------------------#
 
@@ -34,9 +106,29 @@ else
 fi
 
 
-build/install_dependencies
-check_error $? "build/install_dependencies"
+write_log "Installing dependencies [sh -x build/install_dependencies]"
+
+# All jars below are not present in maven central or any public repository, thats why they are added manually while building gimel.
+
+mvn install:install-file -DgroupId=qubole-hive-JDBC -DartifactId=qubole-hive-JDBC -Dversion=0.0.7 -Dpackaging=jar -Dfile=${this_dir}/../lib/qubole-hive-JDBC.jar 1>>/dev/null 2>&1
+check_error $? "install qubole-hive-JDBC"
+
+mvn install:install-file -DgroupId=com.hortonworks -DartifactId=shc-core -Dversion=1.1.2-2.3-s_2.11 -Dpackaging=jar -Dfile=${this_dir}/../lib/shc-core.jar 1>>/dev/null 2>&1
+check_error $? "install shc-core"
+
+mvn install:install-file -DgroupId=com.osscube -DartifactId=aerospike-spark -Dversion=0.3-SNAPSHOT -Dpackaging=jar -Dfile=${this_dir}/../lib/aerospike-spark.jar 1>>/dev/null 2>&1
+check_error $? "install aerospike-spark"
+
+mvn install:install-file -DgroupId=com.twitter -DartifactId=zookeeper-client_2.10 -Dversion=2.0.0_fs-b -Dpackaging=jar -Dfile=${this_dir}/../lib/zookeeper-client_2.10-2.0.0_fs-b.jar 1>>/dev/null 2>&1
+check_error $? "install zookeeper-client"
+
+mvn install:install-file -DgroupId=com.teradata.jdbc -DartifactId=terajdbc4 -Dversion=15.10.00.22 -Dpackaging=jar -Dfile=${this_dir}/../lib/terajdbc4-15.10.00.22.jar 1>>/dev/null 2>&1
+check_error $? "install terajdbc4"
+
+mvn install:install-file -DgroupId=com.teradata.jdbc -DartifactId=tdgssconfig -Dversion=15.10.00.22 -Dpackaging=jar -Dfile=${this_dir}/../lib/tdgssconfig-15.10.00.22.jar 1>>/dev/null 2>&1
+check_error $? "install tdgssconfig"
 
+write_log "Building the project [mvn install ${user_args}]"
 mvn install "$@"
 check_error $? "mvn install $@"
 

diff --git a/build/install_dependencies b/build/install_dependencies
@@ -18,8 +18,77 @@
 
 this_script=`pwd`/$BASH_SOURCE
 this_dir=`dirname $this_script`
-# Source reusable functions
-source ${this_dir}/gimel_functions
+
+#----------------------------function will check for error code & exit if failure, else proceed further----------------------------#
+
+#usage : check_error <$?> <custom_error_code>
+#Example: Check_error < pass $? from the shell command > < Custom Message for errorcode -gt 0 >
+
+check_error()
+{
+	cmd_error_code=$1
+	custom_message=$2
+	if  [ ${cmd_error_code} -gt 0 ]; then
+	  write_log "Error    | Stage |  ${custom_message}"
+	  exit ${cmd_error_code}
+	else
+	  write_log "Success  | Stage | ${custom_message}"
+	fi
+}
+
+#----------------------------function will check for error code & warn if failure----------------------------#
+
+#usage : check_warning <$?> <custom_error_code>
+#Example: Check_warning < pass $? from the shell command > < Custom Message for errorcode -gt 0 >
+
+
+check_warning()
+{
+
+        cmd_error_code=$1
+        pgm_exit_code=$2
+        pgm_exit_msg=$3
+        if  [ ${cmd_error_code} -gt 0 ]; then
+                write_log "WARNING ! ${cmd_error_code} ${pgm_exit_code} ${pgm_exit_msg}"
+        else
+                echo ""
+        fi
+}
+
+
+
+#----------------------------function will write the message to Console / Log File----------------------------#
+
+#Usage : write_log < Whatever message you need to log >
+
+write_log()
+{
+        msg=$1
+        to_be_logged="$(date '+%Y%m%d %H:%M:%S') | $msg"
+        echo ${to_be_logged}
+}
+
+#-----------------------------------Executes a Command--------------------------------------------------------#
+
+
+
+#Usage : run_cmd < The command to execute > 
+
+run_cmd()
+{
+       cmd=$1
+       if [ -z $2 ]; then 
+         fail_on_error="break_code"
+       else
+         fail_on_error=$2
+       fi 
+       write_log "Executing Command --> $1"
+       $cmd
+       error_code=$?
+       if [ ! $fail_on_error = "ignore_errors" ]; then
+           check_error $error_code "$cmd"
+       fi
+}
 
 #--------------------------------Begin execution of Steps------------------------------------------------#
 

diff --git a/gimel-dataapi/gimel-common/pom.xml b/gimel-dataapi/gimel-common/pom.xml
@@ -55,9 +55,9 @@ under the License.
       <scope>${packaging.scope}</scope>
     </dependency>
     <dependency>
-      <groupId>org.scala-lang</groupId>
-      <artifactId>scala-xml</artifactId>
-      <version>2.11.0-M4</version>
+      <groupId>org.scala-lang.modules</groupId>
+      <artifactId>scala-xml_${scala.binary.version}</artifactId>
+      <version>${scala.xml.version}</version>
       <scope>${scala.packaging.scope}</scope>
     </dependency>
     <dependency>
@@ -82,12 +82,6 @@ under the License.
       <groupId>com.paypal.gimel</groupId>
       <artifactId>gimel-logger</artifactId>
       <version>${gimel.version}-SNAPSHOT</version>
-      <exclusions>
-        <exclusion>
-          <groupId>org.apache.kafka</groupId>
-          <artifactId>kafka-clients</artifactId>
-        </exclusion>
-      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.kafka</groupId>
@@ -97,7 +91,7 @@ under the License.
     <dependency>
       <groupId>com.paypal.gimel</groupId>
       <artifactId>serde-common</artifactId>
-      <version>1.0-SNAPSHOT</version>
+      <version>2.4.7-SNAPSHOT</version>
       <scope>${packaging.scope}</scope>
     </dependency>
     <dependency>
@@ -106,12 +100,6 @@ under the License.
       <version>${kafka.version}</version>
       <scope>${packaging.scope}</scope>
     </dependency>
-    <dependency>
-      <groupId>com.databricks</groupId>
-      <artifactId>spark-avro_${scala.binary.version}</artifactId>
-      <version>4.0.0</version>
-      <scope>${packaging.scope}</scope>
-    </dependency>
     <dependency>
       <groupId>org.apache.curator</groupId>
       <artifactId>curator-framework</artifactId>
@@ -162,7 +150,7 @@ under the License.
     </dependency>
     <dependency>
       <groupId>org.apache.spark</groupId>
-      <artifactId>spark-streaming-kafka-${kafka.binary.version}_${scala.binary.version}</artifactId>
+      <artifactId>spark-streaming-kafka-${spark.kafka.connector.version}</artifactId>
       <version>${spark.version}</version>
       <scope>${spark.packaging.scope}</scope>
     </dependency>
@@ -197,7 +185,7 @@ under the License.
       <version>${confluent.version}</version>
       <scope>test</scope>
     </dependency>
-    <!-- Kafka local testing utility needs Netty 3.x at test scope for the minicluster -->
+<!--     Kafka local testing utility needs Netty 3.x at test scope for the minicluster -->
     <dependency>
       <groupId>io.netty</groupId>
       <artifactId>netty</artifactId>
@@ -211,7 +199,7 @@ under the License.
       <scope>test</scope>
     </dependency>
     <dependency>
-      <groupId> net.jpountz.lz4</groupId>
+      <groupId>net.jpountz.lz4</groupId>
       <artifactId>lz4</artifactId>
       <version>1.3.0</version>
       <scope>test</scope>
@@ -220,16 +208,19 @@ under the License.
       <groupId>com.fasterxml.jackson.core</groupId>
       <artifactId>jackson-core</artifactId>
       <version>${jackson.version}</version>
+      <scope>compile</scope>
     </dependency>
     <dependency>
       <groupId>com.fasterxml.jackson.core</groupId>
       <artifactId>jackson-annotations</artifactId>
       <version>${jackson.version}</version>
+      <scope>compile</scope>
     </dependency>
     <dependency>
       <groupId>com.fasterxml.jackson.core</groupId>
       <artifactId>jackson-databind</artifactId>
       <version>${jackson.version}</version>
+      <scope>compile</scope>
     </dependency>
   </dependencies>
 
@@ -277,7 +268,7 @@ under the License.
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-shade-plugin</artifactId>
-        <version>3.2.1</version>
+        <version>${maven.shade.plugin.version}</version>
         <configuration>
           <relocations>
             <relocation>

diff --git a/...l-connectors/gimel-aerospike-3.14/pom.xml → .../gimel-connectors/gimel-aerospike/pom.xml b/...l-connectors/gimel-aerospike-3.14/pom.xml → .../gimel-connectors/gimel-aerospike/pom.xml
@@ -28,7 +28,7 @@ under the License.
   </parent>
   <modelVersion>4.0.0</modelVersion>
 
-  <artifactId>gimel-aerospike-3.14</artifactId>
+  <artifactId>gimel-aerospike</artifactId>
   <version>2.4.7-SNAPSHOT</version>
 
   <dependencies>
@@ -58,7 +58,7 @@ under the License.
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-shade-plugin</artifactId>
-        <version>3.0.0</version>
+        <version>${maven.shade.plugin.version}</version>
         <configuration>
           <relocations>
             <relocation>

diff --git a/.../com/paypal/gimel/aerospike/DataSet.scala → .../com/paypal/gimel/aerospike/DataSet.scala b/.../com/paypal/gimel/aerospike/DataSet.scala → .../com/paypal/gimel/aerospike/DataSet.scala
diff --git a/...e/conf/AerospikeClientConfiguration.scala → ...e/conf/AerospikeClientConfiguration.scala b/...e/conf/AerospikeClientConfiguration.scala → ...e/conf/AerospikeClientConfiguration.scala
diff --git a/...mel/aerospike/conf/AerospikeConfigs.scala → ...mel/aerospike/conf/AerospikeConfigs.scala b/...mel/aerospike/conf/AerospikeConfigs.scala → ...mel/aerospike/conf/AerospikeConfigs.scala
diff --git a/...el/aerospike/reader/AerospikeReader.scala → ...el/aerospike/reader/AerospikeReader.scala b/...el/aerospike/reader/AerospikeReader.scala → ...el/aerospike/reader/AerospikeReader.scala
diff --git a/...ospike/utilities/AeroSparkConnector.scala → ...ospike/utilities/AeroSparkConnector.scala b/...ospike/utilities/AeroSparkConnector.scala → ...ospike/utilities/AeroSparkConnector.scala
diff --git a/...ospike/utilities/AerospikeUtilities.scala → ...ospike/utilities/AerospikeUtilities.scala b/...ospike/utilities/AerospikeUtilities.scala → ...ospike/utilities/AerospikeUtilities.scala
@@ -57,7 +57,7 @@ object AerospikeUtilities {
       val aerospikeSet = conf.aerospikeSet
       val aerospikeRowKey = conf.aerospikeRowKey
       // For each partition of Dataframe, aerospike client is created and is used to write data to aerospike
-      dataFrame.foreachPartition { partition =>
+      dataFrame.foreachPartition { partition: Iterator[Row] =>
         val client = AerospikeAdminClient.createClientConnection(aerospikeHosts, aerospikePort.toInt)
         partition.foreach { row =>
           val bins = columns.map(eachCol => new Bin(eachCol.toString, row.getAs(eachCol).toString)).toArray

diff --git a/...el/aerospike/writer/AerospikeWriter.scala → ...el/aerospike/writer/AerospikeWriter.scala b/...el/aerospike/writer/AerospikeWriter.scala → ...el/aerospike/writer/AerospikeWriter.scala
diff --git a/...el-connectors/gimel-cassandra-2.0/pom.xml → .../gimel-connectors/gimel-cassandra/pom.xml b/...el-connectors/gimel-cassandra-2.0/pom.xml → .../gimel-connectors/gimel-cassandra/pom.xml
@@ -28,7 +28,7 @@ under the License.
   </parent>
 
   <modelVersion>4.0.0</modelVersion>
-  <artifactId>gimel-cassandra-2.0</artifactId>
+  <artifactId>gimel-cassandra</artifactId>
   <version>2.4.7-SNAPSHOT</version>
 
   <dependencies>
@@ -42,11 +42,6 @@ under the License.
       <artifactId>spark-cassandra-connector_${scala.binary.version}</artifactId>
       <version>${cassandra.spark.version}</version>
     </dependency>
-    <dependency>
-      <groupId>com.datastax.spark</groupId>
-      <artifactId>spark-cassandra-connector-java_${scala.binary.version}</artifactId>
-      <version>1.5.2</version>
-    </dependency>
     <dependency>
       <groupId>org.scalatest</groupId>
       <artifactId>scalatest_${scala.binary.version}</artifactId>
@@ -62,7 +57,7 @@ under the License.
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-shade-plugin</artifactId>
-        <version>3.0.0</version>
+        <version>${maven.shade.plugin.version}</version>
         <configuration>
           <relocations>
             <relocation>

diff --git a/.../com/paypal/gimel/cassandra/DataSet.scala → .../com/paypal/gimel/cassandra/DataSet.scala b/.../com/paypal/gimel/cassandra/DataSet.scala → .../com/paypal/gimel/cassandra/DataSet.scala
diff --git a/...a/conf/CassandraClientConfiguration.scala → ...a/conf/CassandraClientConfiguration.scala b/...a/conf/CassandraClientConfiguration.scala → ...a/conf/CassandraClientConfiguration.scala
diff --git a/...mel/cassandra/conf/CassandraConfigs.scala → ...mel/cassandra/conf/CassandraConfigs.scala b/...mel/cassandra/conf/CassandraConfigs.scala → ...mel/cassandra/conf/CassandraConfigs.scala
diff --git a/...el/cassandra/reader/CassandraReader.scala → ...el/cassandra/reader/CassandraReader.scala b/...el/cassandra/reader/CassandraReader.scala → ...el/cassandra/reader/CassandraReader.scala
diff --git a/...el/cassandra/writer/CassandraWriter.scala → ...el/cassandra/writer/CassandraWriter.scala b/...el/cassandra/writer/CassandraWriter.scala → ...el/cassandra/writer/CassandraWriter.scala
diff --git a/...gimel-connectors/gimel-druid-0.82/pom.xml → ...aapi/gimel-connectors/gimel-druid/pom.xml b/...gimel-connectors/gimel-druid-0.82/pom.xml → ...aapi/gimel-connectors/gimel-druid/pom.xml
@@ -40,13 +40,17 @@ under the License.
     <!--Druid Libraries-->
     <dependency>
       <groupId>io.druid</groupId>
-      <artifactId>tranquility-core_${scala.binary.version}</artifactId>
+      <artifactId>tranquility-core_2.11</artifactId>
       <version>${tranquility.version}</version>
       <exclusions>
         <exclusion>
           <groupId>com.fasterxml.jackson.core</groupId>
           <artifactId>jackson-databind</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>org.scala-lang</groupId>
+          <artifactId>*</artifactId>
+        </exclusion>
         <exclusion>
           <groupId>com.fasterxml.jackson.core</groupId>
           <artifactId>jackson-core</artifactId>

diff --git a/...cala/com/paypal/gimel/druid/DataSet.scala → ...cala/com/paypal/gimel/druid/DataSet.scala b/...cala/com/paypal/gimel/druid/DataSet.scala → ...cala/com/paypal/gimel/druid/DataSet.scala
diff --git a/...druid/conf/DruidClientConfiguration.scala → ...druid/conf/DruidClientConfiguration.scala b/...druid/conf/DruidClientConfiguration.scala → ...druid/conf/DruidClientConfiguration.scala
diff --git a/...aypal/gimel/druid/conf/DruidConfigs.scala → ...aypal/gimel/druid/conf/DruidConfigs.scala b/...aypal/gimel/druid/conf/DruidConfigs.scala → ...aypal/gimel/druid/conf/DruidConfigs.scala
diff --git a/...pal/gimel/druid/conf/DruidConstants.scala → ...pal/gimel/druid/conf/DruidConstants.scala b/...pal/gimel/druid/conf/DruidConstants.scala → ...pal/gimel/druid/conf/DruidConstants.scala
diff --git a/...al/gimel/druid/model/DruidDimension.scala → ...al/gimel/druid/model/DruidDimension.scala b/...al/gimel/druid/model/DruidDimension.scala → ...al/gimel/druid/model/DruidDimension.scala
diff --git a/...aypal/gimel/druid/model/DruidMetric.scala → ...aypal/gimel/druid/model/DruidMetric.scala b/...aypal/gimel/druid/model/DruidMetric.scala → ...aypal/gimel/druid/model/DruidMetric.scala
diff --git a/...ypal/gimel/druid/reader/DruidReader.scala → ...ypal/gimel/druid/reader/DruidReader.scala b/...ypal/gimel/druid/reader/DruidReader.scala → ...ypal/gimel/druid/reader/DruidReader.scala
diff --git a/...pal/gimel/druid/util/DruidEventBeam.scala → ...pal/gimel/druid/util/DruidEventBeam.scala b/...pal/gimel/druid/util/DruidEventBeam.scala → ...pal/gimel/druid/util/DruidEventBeam.scala
diff --git a/...aypal/gimel/druid/util/DruidUtility.scala → ...aypal/gimel/druid/util/DruidUtility.scala b/...aypal/gimel/druid/util/DruidUtility.scala → ...aypal/gimel/druid/util/DruidUtility.scala
diff --git a/...el/druid/writer/DruidRealtimeWriter.scala → ...el/druid/writer/DruidRealtimeWriter.scala b/...el/druid/writer/DruidRealtimeWriter.scala → ...el/druid/writer/DruidRealtimeWriter.scala
diff --git a/...ypal/gimel/druid/writer/DruidWriter.scala → ...ypal/gimel/druid/writer/DruidWriter.scala b/...ypal/gimel/druid/writer/DruidWriter.scala → ...ypal/gimel/druid/writer/DruidWriter.scala