Fix broken CI tests and test release code (#894)

* Release branch test

* Add v to version number

* add back some missing dependencies

* Add snowflake, cosmos dependencies to buildscript dependency

* Fix snowflake version

* Add snowflake and cosmos as a dependency

* Add few more missing dependencies

* Remove META-INF

* Add sql server dependency

* Add spark-core 3.1 dependency

* add spark sql kafka

* Add jetty

* add compileOnly implementations

* Latest version of cosmos

* Latest version of cosmos

* add sqllite dependency

* upgrade to spark 3.2

* Experiment by changing to only provided

* Change to implementation of snowflake componenets

* update version

* Align all implementations before provided in gradle file

Co-authored-by: rkashyap <[email protected]>

.gitignore

@@ -217,6 +217,7 @@ build
 .bloop/
 project/.bloop
 metals.sbt
+feathr-data-models/src/mainGeneratedDataTemplate/
 
 .bsp/sbt.json
 

build.gradle

@@ -55,6 +55,8 @@ jar {
   from {
     configurations.runtimeClasspath.collect { it.isDirectory() ? it : zipTree(it) }
   }
+  exclude 'META-INF/*.RSA', 'META-INF/*.SF','META-INF/*.DSA'
+  zip64 = true
 }
 
 dependencies {
@@ -64,6 +66,16 @@ dependencies {
   implementation project(":feathr-impl")
   // needed to include data models in jar
   extraLibs project(path: ':feathr-data-models', configuration: 'dataTemplate')
+  implementation 'net.snowflake:snowflake-jdbc:3.13.18'
+  implementation 'net.snowflake:spark-snowflake_2.12:2.10.0-spark_3.2'
+  provided 'com.microsoft.azure:azure-eventhubs-spark_2.12:2.3.21'
+  provided 'com.azure.cosmos.spark:azure-cosmos-spark_3-2_2-12:4.11.1'
+  provided 'com.microsoft.sqlserver:mssql-jdbc:10.2.0.jre8'
+  provided 'org.eclipse.jetty:jetty-util:9.3.24.v20180605'
+  provided 'org.apache.kafka:kafka-clients:3.1.0'
+  provided 'org.apache.spark:spark-core_2.12:3.1.3'
+  provided 'org.apache.spark:spark-sql-kafka-0-10_2.12:3.1.3'
+  provided 'org.postgresql:postgresql:42.3.4'
 }
 
 ext {
@@ -105,22 +117,31 @@ project.ext.spec = [
             'spark_hive' : "org.apache.spark:spark-hive_$ver.scala_rt:$ver.spark",
             'spark_sql' : "org.apache.spark:spark-sql_$ver.scala_rt:$ver.spark",
             'spark_catalyst' : "org.apache.spark:spark-catalyst_$ver.scala_rt:$ver.spark",
+            "spark_sql_kafka" : "org.apache.spark:spark-sql-kafka-0-10_$ver.scala_rt:3.1.3"
         ],
         'scala' : [
             'scala_library' : "org.scala-lang:scala-library:$ver.scala",
             'scalatest' : "org.scalatest:scalatest_$ver.scala_rt:3.0.0",
         ],
         'avro' : "org.apache.avro:avro:1.10.2",
         "avroUtil": "com.linkedin.avroutil1:helper-all:0.2.100",
+        "azure": "com.microsoft.azure:azure-eventhubs-spark_2.12:2.3.21",
         'fastutil' : "it.unimi.dsi:fastutil:8.1.1",
         'mvel' : "org.mvel:mvel2:2.2.8.Final",
-        'protobuf' : "com.google.protobuf:protobuf-java:3.19.4",
+        'protobuf' : "com.google.protobuf:protobuf-java:2.6.1",
         'guava' : "com.google.guava:guava:25.0-jre",
         'xbean' : "org.apache.xbean:xbean-asm6-shaded:4.10",
         'log4j' : "log4j:log4j:1.2.17",
+        'jetty': "org.eclipse.jetty:jetty-util:9.3.24.v20180605",
+        'kafka': "org.apache.kafka:kafka-clients:3.1.0",
+
         'json' : "org.json:json:20180130",
+        'sqlserver': "com.microsoft.sqlserver:mssql-jdbc:10.2.0.jre8",
+        'postgresql': "org.postgresql:postgresql:42.3.4",
         'equalsverifier' : "nl.jqno.equalsverifier:equalsverifier:3.1.12",
         'mockito' : "org.mockito:mockito-core:3.1.0",
+        'snowflake-jdbc' : "net.snowflake:3.13.18",
+        "spark-snowflake_2.12" : "net.snowflake:2.10.0-spark_3.2",
         "mockito_inline": "org.mockito:mockito-inline:2.28.2",
         'testing' : "org.testng:testng:6.14.3",
         'jdiagnostics' : "org.anarres.jdiagnostics:jdiagnostics:1.0.7",

feathr-impl/build.gradle

@@ -56,11 +56,11 @@ dependencies {
   implementation spec.product.guava
   implementation spec.product.xbean
   implementation spec.product.json
-  implementation spec.product.avroUtil
   implementation spec.product.antlr
   implementation spec.product.antlrRuntime
 
   implementation spec.product.jackson.jackson_databind
+  provided spec.product.avroUtil
   provided spec.product.typesafe_config
   provided spec.product.log4j
   provided spec.product.hadoop.common

gradle.properties

@@ -1,3 +1,3 @@
-version=0.11.1-rc1
+version=v0.10.3-rc5
 SONATYPE_AUTOMATIC_RELEASE=true
 POM_ARTIFACT_ID=feathr_2.12