[paypal#247] [spark] Bump Spark Version to 2.4.7 | Bump Spark Version…

… to 2.4.7 along with other stack to align with gcp_dataproc_1.5.x
Dee-Pac · Nov 7, 2020 · a83d31f · a83d31f
1 parent e957499
commit a83d31f
Show file tree

Hide file tree

Showing 5 changed files with 30 additions and 9 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -29,4 +29,4 @@ before_install:
 
 # Build
 install: 
-- sh build/gimel -Dmaven.test.skip=true -pl gimel-dataapi/gimel-tools -am | egrep -v "Download|Copy|Including|WARNING"
+- sh -x build/gimel -Dmaven.test.skip=true -pl gimel-dataapi/gimel-tools -am | egrep -v "Download|Copy|Including|WARNING"
diff --git a/build/gimel b/build/gimel
@@ -15,6 +15,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+echo "Present Working Dir"
+echo $PWD
+
 this_dir=`dirname build/install_dependencies`
 
 #----------------------------function will check for error code & exit if failure, else proceed further----------------------------#
@@ -103,9 +106,29 @@ else
 fi
 
 
-build/install_dependencies
-check_error $? "build/install_dependencies"
+write_log "Installing dependencies [sh -x build/install_dependencies]"
+
+# All jars below are not present in maven central or any public repository, thats why they are added manually while building gimel.
+
+mvn install:install-file -DgroupId=qubole-hive-JDBC -DartifactId=qubole-hive-JDBC -Dversion=0.0.7 -Dpackaging=jar -Dfile=${this_dir}/../lib/qubole-hive-JDBC.jar 1>>/dev/null 2>&1
+check_error $? "install qubole-hive-JDBC"
+
+mvn install:install-file -DgroupId=com.hortonworks -DartifactId=shc-core -Dversion=1.1.2-2.3-s_2.11 -Dpackaging=jar -Dfile=${this_dir}/../lib/shc-core.jar 1>>/dev/null 2>&1
+check_error $? "install shc-core"
+
+mvn install:install-file -DgroupId=com.osscube -DartifactId=aerospike-spark -Dversion=0.3-SNAPSHOT -Dpackaging=jar -Dfile=${this_dir}/../lib/aerospike-spark.jar 1>>/dev/null 2>&1
+check_error $? "install aerospike-spark"
+
+mvn install:install-file -DgroupId=com.twitter -DartifactId=zookeeper-client_2.10 -Dversion=2.0.0_fs-b -Dpackaging=jar -Dfile=${this_dir}/../lib/zookeeper-client_2.10-2.0.0_fs-b.jar 1>>/dev/null 2>&1
+check_error $? "install zookeeper-client"
+
+mvn install:install-file -DgroupId=com.teradata.jdbc -DartifactId=terajdbc4 -Dversion=15.10.00.22 -Dpackaging=jar -Dfile=${this_dir}/../lib/terajdbc4-15.10.00.22.jar 1>>/dev/null 2>&1
+check_error $? "install terajdbc4"
+
+mvn install:install-file -DgroupId=com.teradata.jdbc -DartifactId=tdgssconfig -Dversion=15.10.00.22 -Dpackaging=jar -Dfile=${this_dir}/../lib/tdgssconfig-15.10.00.22.jar 1>>/dev/null 2>&1
+check_error $? "install tdgssconfig"
 
+write_log "Building the project [mvn install ${user_args}]"
 mvn install "$@"
 check_error $? "mvn install $@"
 

diff --git a/gimel-dataapi/gimel-connectors/gimel-restapi/pom.xml b/gimel-dataapi/gimel-connectors/gimel-restapi/pom.xml
@@ -47,12 +47,11 @@ under the License.
 
   <build>
     <sourceDirectory>src/main/scala</sourceDirectory>
-    <testSourceDirectory>src/test/scala</testSourceDirectory>
     <plugins>
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-shade-plugin</artifactId>
-        <version>3.0.0</version>
+        <version>${maven.shade.plugin.version}</version>
         <configuration>
           <relocations>
             <relocation>

diff --git a/gimel-dataapi/gimel-connectors/gimel-s3/pom.xml b/gimel-dataapi/gimel-connectors/gimel-s3/pom.xml
@@ -69,7 +69,6 @@ under the License.
 
   <build>
     <sourceDirectory>src/main/scala</sourceDirectory>
-    <testSourceDirectory>src/test/scala</testSourceDirectory>
     <plugins>
       <plugin>
         <groupId>net.alchim31.maven</groupId>
@@ -111,7 +110,7 @@ under the License.
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-shade-plugin</artifactId>
-        <version>3.0.0</version>
+        <version>${maven.shade.plugin.version}</version>
         <configuration>
           <relocations>
             <relocation>

diff --git a/gimel-dataapi/pom.xml b/gimel-dataapi/pom.xml
@@ -41,14 +41,14 @@ under the License.
         <module>gimel-connectors/gimel-sftp</module>
         <module>gimel-connectors/gimel-elasticsearch</module>
         <module>gimel-connectors/gimel-hive</module>
+        <module>gimel-connectors/gimel-restapi</module>
+        <module>gimel-connectors/gimel-s3</module>
         <module>gimel-connectors/gimel-jdbc</module>
         <module>gimel-connectors/gimel-hbase</module>
         <module>gimel-connectors/gimel-cassandra-2.0</module>
         <module>gimel-connectors/gimel-aerospike-3.14</module>
         <module>gimel-connectors/gimel-kafka-2.2</module>
         <module>gimel-connectors/gimel-druid-0.82</module>
-        <module>gimel-connectors/gimel-restapi</module>
-        <module>gimel-connectors/gimel-s3</module>
         <module>gimel-core</module>
         <module>gimel-sql</module>
         <module>gimel-tools</module>