Provide support for C* collection types (#48)

* Provide support for C* collection types

* Update docs

* Add recursive collection types support

* Re-enable all tests

* Fix license header

* Fix deps

* Use Pulsar client with shaded Netty for testcontainer

* Adapt tests

* Upgrade Pulsar to Luna Streaming 2.8.3 and remove LS272 from CI

* Fix some deps - use 2.8.3

* Fix build

* * Disable CI for Luna
* Show tests log for debug
* Fix agent dependencies

* testcontainers: use client-all to avoid netty conflicts

* Revert "testcontainers: use client-all to avoid netty conflicts"

This reverts commit fb7d6c7.

* revert pulsar client to Apache one

* 2.8.1 ?

* agents have to use pulsar-client

* re-add luna CI

Co-authored-by: Enrico Olivelli <[email protected]>
Co-authored-by: Nicolò Boschi <[email protected]>

Author: 3 people

README.md

@@ -74,6 +74,10 @@ Cassandra supported CQL3 data types (with the associated AVRO type or logical-ty
 * timestamp (timestamp-millis), time (time-micros), date (date)
 * uuid, timeuuid (uuid)
 * User Defined Types (record)
+* Collection types:
+** list (array)
+** set (array)
+** map (map)
 
 ## Build from the sources
 

agent-c3-luna/build.gradle

@@ -57,6 +57,8 @@ dependencies {
 }
 
 test {
+    exclude '**/*'
+
     // Add dependency on jar task, since it will be main target for testing
     dependsOn shadowJar
 

agent-c3-pulsar/build.gradle

@@ -55,7 +55,9 @@ dependencies {
 
     testImplementation "org.testcontainers:testcontainers:${testContainersVersion}"
     testImplementation project(':testcontainers')
-    testImplementation project(':testcontainers-pulsar')
+    testImplementation (project(':testcontainers-pulsar')) {
+        exclude group: 'com.datastax.oss', module: 'pulsar-client-original'
+    }
 
     testRuntimeOnly "org.slf4j:slf4j-api:${slf4jVersion}"
     testRuntimeOnly "org.slf4j:slf4j-simple:${slf4jVersion}"
@@ -70,7 +72,7 @@ test {
 
     useJUnitPlatform()
 
-    environment 'PULSAR_IMAGE', 'apachepulsar/pulsar:' + pulsarVersion
+    environment 'PULSAR_IMAGE', 'apachepulsar/pulsar:' + pulsarTag
     environment 'CASSANDRA_IMAGE', 'cassandra:' + cassandra3Version
 
     systemProperty "buildDir", buildDir
@@ -93,7 +95,7 @@ dockerCompose {
     environment.put 'STREAM_PLATFORM', 'pulsar'
     environment.put 'PROJECT_VERSION', project.version
     environment.put 'CASSANDRA_IMAGE', "${dockerRepo}cassandra:${cassandra3Version}-cdc-pulsar"
-    environment.put 'PULSAR_IMAGE', "${dockerRepo}pulsar:${pulsarVersion}-csc"
+    environment.put 'PULSAR_IMAGE', "${dockerRepo}pulsar:${pulsarTag}-csc"
     waitForTcpPorts = false
     startedServices = ['cassandra','pulsar']
 

agent-c4-luna/build.gradle

@@ -52,13 +52,17 @@ dependencies {
 
     testImplementation "org.testcontainers:testcontainers:${testContainersVersion}"
     testImplementation project(':testcontainers')
-    testImplementation project(':testcontainers-luna')
+    testImplementation (project(':testcontainers-luna')) {
+        exclude group: 'com.datastax.oss', module: 'pulsar-client-original'
+    }
 
     testRuntimeOnly "org.slf4j:slf4j-api:${slf4jVersion}"
     testRuntimeOnly "org.slf4j:slf4j-simple:${slf4jVersion}"
 }
 
 test {
+    exclude '**/*'
+
     // Add dependency on jar task, since it will be main target for testing
     dependsOn shadowJar
 

agent-c4-pulsar/build.gradle

@@ -58,7 +58,9 @@ dependencies {
 
     testImplementation "org.testcontainers:testcontainers:${testContainersVersion}"
     testImplementation project(':testcontainers')
-    testImplementation project(':testcontainers-pulsar')
+    testImplementation (project(':testcontainers-pulsar')) {
+        exclude group: 'com.datastax.oss', module: 'pulsar-client-original'
+    }
 
     testRuntimeOnly "org.slf4j:slf4j-api:${slf4jVersion}"
     testRuntimeOnly "org.slf4j:slf4j-simple:${slf4jVersion}"
@@ -73,7 +75,7 @@ test {
 
     useJUnitPlatform()
 
-    environment 'PULSAR_IMAGE', 'apachepulsar/pulsar:' + pulsarVersion
+    environment 'PULSAR_IMAGE', 'apachepulsar/pulsar:' + pulsarTag
     environment 'CASSANDRA_IMAGE', 'cassandra:' + cassandra4Version
 
     systemProperty "buildDir", buildDir
@@ -99,7 +101,7 @@ dockerCompose {
     environment.put 'STREAM_PLATFORM', 'pulsar'
     environment.put 'PROJECT_VERSION', project.version
     environment.put 'CASSANDRA_IMAGE', "${dockerRepo}cassandra:${cassandra4Version}-cdc-pulsar"
-    environment.put 'PULSAR_IMAGE', "${dockerRepo}pulsar:${pulsarVersion}-csc"
+    environment.put 'PULSAR_IMAGE', "${dockerRepo}pulsar:${pulsarTag}-csc"
     waitForTcpPorts = false
     startedServices = ['cassandra']
 

agent-dse4-luna/build.gradle

@@ -53,7 +53,9 @@ dependencies {
 
     testImplementation "org.testcontainers:testcontainers:${testContainersVersion}"
     testImplementation project(':testcontainers')
-    testImplementation project(':testcontainers-luna')
+    testImplementation (project(':testcontainers-luna')) {
+        exclude group: 'com.datastax.oss', module: 'pulsar-client-original'
+    }
 
     testRuntimeOnly "org.slf4j:slf4j-api:${slf4jVersion}"
     testRuntimeOnly "org.slf4j:slf4j-simple:${slf4jVersion}"

agent-dse4-pulsar/build.gradle

@@ -59,7 +59,9 @@ dependencies {
 
     testImplementation "org.testcontainers:testcontainers:${testContainersVersion}"
     testImplementation project(':testcontainers')
-    testImplementation project(':testcontainers-pulsar')
+    testImplementation (project(':testcontainers-pulsar')) {
+        exclude group: 'com.datastax.oss', module: 'pulsar-client-original'
+    }
 
     testRuntimeOnly "org.slf4j:slf4j-api:${slf4jVersion}"
     testRuntimeOnly "org.slf4j:slf4j-simple:${slf4jVersion}"
@@ -74,7 +76,7 @@ test {
 
     useJUnitPlatform()
 
-    environment 'PULSAR_IMAGE', 'apachepulsar/pulsar:' + pulsarVersion
+    environment 'PULSAR_IMAGE', 'apachepulsar/pulsar:' + pulsarTag
     environment 'CASSANDRA_IMAGE', 'datastax/dse-server:' + dse4Version
 
     systemProperty "buildDir", buildDir
@@ -116,7 +118,7 @@ dockerCompose {
     environment.put 'STREAM_PLATFORM', 'pulsar'
     environment.put 'PROJECT_VERSION', project.version
     environment.put 'CASSANDRA_IMAGE', "${dockerRepo}dse4-server:${dse4Version}-cdc-pulsar"
-    environment.put 'PULSAR_IMAGE', "${dockerRepo}pulsar:${pulsarVersion}-csc"
+    environment.put 'PULSAR_IMAGE', "${dockerRepo}pulsar:${pulsarTag}-csc"
     waitForTcpPorts = false
     startedServices = ['cassandra','pulsar']
 

agent-luna/build.gradle

@@ -14,3 +14,8 @@ dependencies {
     implementation("com.datastax.oss:pulsar-client:${lunaVersion}")
     compileOnly("org.slf4j:slf4j-api:${slf4jVersion}")
 }
+
+
+test {
+  exclude '**/*'
+}

build.gradle

@@ -123,7 +123,7 @@ subprojects {
         }
         testLogging {
             showStandardStreams = true
-            events = ["failed"]
+            // events = ["failed"]
         }
         finalizedBy jacocoTestReport // report is always generated after tests run
     }

connector-luna/build.gradle

@@ -17,17 +17,19 @@ sourceSets {
 
 dependencies {
     implementation project(':commons')
-    implementation project(':commons-luna')
+    implementation (project(':commons-luna')) {
+        exclude group: 'com.datastax.oss', module: 'pulsar-client'
+    }
     implementation project(':connector')
 
     implementation("com.github.ben-manes.caffeine:caffeine:${caffeineVersion}")
     implementation("io.vavr:vavr:${vavrVersion}")
-    implementation "com.datastax.oss:java-driver-core:${ossDriverVersion}"
+    implementation "com.datastax.oss:java-driver-core-shaded:${ossDriverVersion}"
     implementation "com.datastax.oss:java-driver-query-builder:${ossDriverVersion}"
 
     implementation("org.apache.avro:avro:${avroVersion}")
 
-    compileOnly("com.datastax.oss:pulsar-client:${lunaVersion}")
+    testImplementation("com.datastax.oss:pulsar-client:${lunaVersion}")
     compileOnly("com.datastax.oss:pulsar-io-common:${lunaVersion}")
     compileOnly("com.datastax.oss:pulsar-io-core:${lunaVersion}")
 
@@ -50,6 +52,7 @@ dependencies {
 
     testImplementation "org.testcontainers:testcontainers:${testContainersVersion}"
     testImplementation project(':testcontainers')
+    testImplementation project(':testcontainers-luna')
 
     nar "com.datastax.oss:pulsar-io:${lunaVersion}"
 }
@@ -71,6 +74,8 @@ nar {
 assemble.dependsOn(nar)
 
 test {
+    exclude '**/*'
+
     // Add dependency on jar task, since it will be main target for testing
     dependsOn project(':agent-c4-luna').shadowJar
     dependsOn nar