From 1fb367358fe963f71de21e38750978f0fcd1be35 Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Fri, 10 Jun 2022 20:42:56 +0530 Subject: [PATCH] [SNAPPYDATA] updated dependencies to address major CVEs - also exclude log4j2 from yarn shuffle shadow jar - fixed occasional failures in kafka suite --- build.gradle | 25 +++++++++++-------- common/network-yarn/build.gradle | 2 +- core/build.gradle | 16 +++++++----- .../docker-integration-tests/build.gradle | 2 +- .../spark/sql/kafka010/KafkaSourceSuite.scala | 14 +++++------ sql/hive/build.gradle | 2 +- yarn/build.gradle | 3 ++- 7 files changed, 36 insertions(+), 28 deletions(-) diff --git a/build.gradle b/build.gradle index 869abadd79b8f..3b5816413f1ea 100644 --- a/build.gradle +++ b/build.gradle @@ -60,30 +60,29 @@ allprojects { scalaBinaryVersion = '2.11' scalaVersion = scalaBinaryVersion + '.8' hadoopVersion = '3.2.0' - protobufVersion = '3.6.1' - jerseyVersion = '2.22.2' + protobufVersion = '3.16.1' + jerseyVersion = '2.35' sunJerseyVersion = '1.19.4' jettyVersion = '9.4.43.v20210629' yarnTestJettyVersion = '9.3.24.v20180605' // only used by yarn tests jettyOldVersion = '6.1.26' log4j2Version = '2.17.2' slf4jVersion = '1.7.32' - junitVersion = '4.12' + junitVersion = '4.13.2' mockitoVersion = '1.10.19' javaxServletVersion = '3.1.0' guavaVersion = '14.0.1' hiveVersion = '1.21.2.7.0.3.2-3' chillVersion = '0.8.5' kryoVersion = '4.0.2' - nettyVersion = '3.10.6.Final' - nettyAllVersion = '4.1.73.Final' + nettyVersion = '4.1.77.Final' derbyVersion = '10.14.2.0' - httpClientVersion = '4.5.6' + httpClientVersion = '4.5.13' httpCoreVersion = '4.4.10' levelDbJniVersion = '1.8' jackson1Version = '1.9.14-atlassian-6' - jacksonVersion = '2.13.1' - jacksonDatabindVersion = '2.13.1' + jacksonVersion = '2.13.3' + jacksonDatabindVersion = '2.13.3' snappyJavaVersion = '1.1.8.4' lz4Version = '1.8.0' lzfVersion = '1.1' @@ -92,11 +91,13 @@ allprojects { metricsVersion = '3.2.6' janinoVersion = '3.0.8' thriftVersion = '0.9.3' + fb303Version = '0.9.3' antlrVersion = '4.5.3' jpamVersion = '1.1' seleniumVersion = '2.52.0' curatorVersion = '2.7.1' - commonsCodecVersion = '1.11' + commonsCodecVersion = '1.15' + commonsCompressVersion = '1.21' commonsCryptoVersion = '1.0.0' commonsLang3Version = '3.8.1' commonsMath3Version = '3.6.1' @@ -120,7 +121,7 @@ allprojects { dockerClientVersion = '8.14.5' mysqlVersion = '8.0.13' postgresqlVersion = '42.2.5' - ojdbc6Version = '11.2.0.4' + ojdbc8Version = '12.2.0.1' zookeeperVersion = '3.6.2' activationVersion = '1.1.1' roaringBitmapVersion = '0.6.66' @@ -181,7 +182,9 @@ allprojects { exclude(group: 'org.apache.ant', module: 'ant') exclude(group: 'com.esotericsoftware.kryo', module: 'kryo') exclude(group: 'commons-codec', module: 'commons-codec') + exclude(group: 'org.apache.commons', module: 'commons-compress') exclude(group: 'commons-httpclient', module: 'commons-httpclient') + exclude(group: 'org.apache.avro', module: 'avro') exclude(group: 'org.apache.avro', module: 'avro-mapred') exclude(group: 'org.apache.calcite', module: 'calcite-core') exclude(group: 'org.apache.curator', module: 'apache-curator') @@ -232,7 +235,7 @@ allprojects { exclude(group: 'org.apache.velocity', module: 'velocity') exclude(group: 'org.apache.avro', module: 'avro-ipc') }, - netty_all: dependencies.create("io.netty:netty-all:${nettyAllVersion}") { + netty_all: dependencies.create("io.netty:netty-all:${nettyVersion}") { exclude(group: 'io.netty', module: 'netty-codec-dns') exclude(group: 'io.netty', module: 'netty-codec-haproxy') exclude(group: 'io.netty', module: 'netty-codec-http') diff --git a/common/network-yarn/build.gradle b/common/network-yarn/build.gradle index 9c537cad86ef1..788c86a11a9fe 100644 --- a/common/network-yarn/build.gradle +++ b/common/network-yarn/build.gradle @@ -41,7 +41,7 @@ shadowJar { exclude(dependency('org.scala-lang:.*')) exclude(dependency('org.scala-lang.modules:.*')) exclude(dependency('org.slf4j:.*')) - exclude(dependency('log4j:.*')) + exclude(dependency('org.apache.logging.log4j:.*')) exclude(dependency('org.scalatest:.*')) } //configurations = [ project.configurations.runtimeJar ] diff --git a/core/build.gradle b/core/build.gradle index 8f078c0508812..b243fc4895049 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -24,6 +24,7 @@ dependencies { compile project(subprojectBase + 'snappy-spark-unsafe_' + scalaBinaryVersion) compile project(subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion) + compile group: 'org.apache.commons', name: 'commons-compress', version: commonsCompressVersion compile libraries.avro_ipc compile libraries.avro_mapred compile group: 'com.google.guava', name: 'guava', version: guavaVersion @@ -35,14 +36,11 @@ dependencies { exclude(group: 'com.esotericsoftware', module: 'kryo-shaded') } compile group: 'org.apache.xbean', name: 'xbean-asm5-shaded', version: xbeanAsm5Version - // explicitly include netty from akka-remote to not let zookeeper override it - compile(group: 'io.netty', name: 'netty', version: nettyVersion) { - exclude(group: 'log4j', module: 'log4j') - } // explicitly exclude old netty from zookeeper compile(group: 'org.apache.zookeeper', name: 'zookeeper', version: zookeeperVersion) { exclude(group: 'org.jboss.netty', module: 'netty') exclude(group: 'jline', module: 'jline') + exclude(group: 'io.netty', module: 'netty') exclude(group: 'io.netty', module: 'netty-handler') exclude(group: 'io.netty', module: 'netty-transport-native-epoll') exclude(group: 'com.github.spotbugs', module: 'spotbugs-annotations') @@ -93,10 +91,15 @@ dependencies { compile group: 'commons-net', name: 'commons-net', version: commonsNetVersion compile group: 'org.json4s', name: 'json4s-jackson_' + scalaBinaryVersion, version: json4sVersion compile group: 'org.glassfish.jersey.core', name: 'jersey-client', version: jerseyVersion - compile group: 'org.glassfish.jersey.core', name: 'jersey-common', version: jerseyVersion - compile group: 'org.glassfish.jersey.core', name: 'jersey-server', version: jerseyVersion + compile(group: 'org.glassfish.jersey.core', name: 'jersey-common', version: jerseyVersion) { + exclude(group: 'com.sun.activation', module: 'jakarta.activation') + } + compile(group: 'org.glassfish.jersey.core', name: 'jersey-server', version: jerseyVersion) { + exclude(group: 'jakarta.xml.bind', module: 'jakarta.xml.bind-api') + } compile group: 'org.glassfish.jersey.containers', name: 'jersey-container-servlet', version: jerseyVersion compile group: 'org.glassfish.jersey.containers', name: 'jersey-container-servlet-core', version: jerseyVersion + compile group: 'org.glassfish.jersey.inject', name: 'jersey-hk2', version: jerseyVersion compile(group: 'com.clearspring.analytics', name: 'stream', version: streamVersion) { exclude(group: 'it.unimi.dsi', module: 'fastutil') } @@ -137,6 +140,7 @@ dependencies { exclude(group: 'com.google.guava', module: 'guava') exclude(group: 'io.netty', module: 'netty') } + testCompile group: 'org.glassfish.jersey.test-framework.providers', name: 'jersey-test-framework-provider-simple', version: jerseyVersion testCompile(group: 'org.seleniumhq.selenium', name: 'selenium-htmlunit-driver', version: seleniumVersion) { exclude(group: 'com.google.guava', module: 'guava') } diff --git a/external/docker-integration-tests/build.gradle b/external/docker-integration-tests/build.gradle index 526f0a7bb4d78..d53b5df9cbae5 100644 --- a/external/docker-integration-tests/build.gradle +++ b/external/docker-integration-tests/build.gradle @@ -37,7 +37,7 @@ dependencies { testCompile group: 'org.apache.httpcomponents', name: 'httpcore', version: httpCoreVersion testCompile group: 'mysql', name: 'mysql-connector-java', version: mysqlVersion testCompile group: 'org.postgresql', name: 'postgresql', version: postgresqlVersion - testCompile group: 'com.oracle.database.jdbc', name: 'ojdbc6', version: ojdbc6Version + testCompile group: 'com.oracle.database.jdbc', name: 'ojdbc8', version: ojdbc8Version testCompile group: 'com.sun.jersey', name: 'jersey-server', version: sunJerseyVersion testCompile group: 'com.sun.jersey', name: 'jersey-core', version: sunJerseyVersion testCompile group: 'com.sun.jersey', name: 'jersey-servlet', version: sunJerseyVersion diff --git a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaSourceSuite.scala b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaSourceSuite.scala index fcc1f6a8c722c..0e1e8b2e62073 100644 --- a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaSourceSuite.scala +++ b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaSourceSuite.scala @@ -432,7 +432,7 @@ class KafkaSourceSuite extends KafkaSourceTest { .format("kafka") .option("kafka.bootstrap.servers", testUtils.brokerAddress) .option("kafka.metadata.max.age.ms", "1") - .option("kafka.default.api.timeout.ms", "3000") + .option("kafka.default.api.timeout.ms", "10000") .option("subscribePattern", s"$topicPrefix-.*") .option("failOnDataLoss", "false") @@ -577,16 +577,16 @@ class KafkaSourceSuite extends KafkaSourceTest { .readStream .format("kafka") .option("kafka.bootstrap.servers", testUtils.brokerAddress) - .option("kafka.default.api.timeout.ms", "3000") + .option("kafka.default.api.timeout.ms", "10000") .option("kafka.metadata.max.age.ms", "1") .option("subscribe", topic) // If a topic is deleted and we try to poll data starting from offset 0, // the Kafka consumer will just block until timeout and return an empty result. // So set the timeout to 1 second to make this test fast. - .option("kafkaConsumer.pollTimeoutMs", "1000") + .option("kafkaConsumer.pollTimeoutMs", "3000") .option("startingOffsets", "earliest") .option("failOnDataLoss", "false") - .option("kafka.default.api.timeout.ms", "3000") + .option("kafka.default.api.timeout.ms", "10000") val kafka = reader.load() .selectExpr("CAST(key AS STRING)", "CAST(value AS STRING)") .as[(String, String)] @@ -888,7 +888,7 @@ class KafkaSourceStressSuite extends KafkaSourceTest { .option("kafka.metadata.max.age.ms", "1") .option("subscribePattern", "stress.*") .option("failOnDataLoss", "false") - .option("kafka.default.api.timeout.ms", "3000") + .option("kafka.default.api.timeout.ms", "10000") .load() .selectExpr("CAST(key AS STRING)", "CAST(value AS STRING)") .as[(String, String)] @@ -985,11 +985,11 @@ class KafkaSourceStressForDontFailOnDataLossSuite extends StreamTest with Shared .format("kafka") .option("kafka.bootstrap.servers", testUtils.brokerAddress) .option("kafka.metadata.max.age.ms", "1") - .option("kafka.default.api.timeout.ms", "3000") + .option("kafka.default.api.timeout.ms", "10000") .option("subscribePattern", "failOnDataLoss.*") .option("startingOffsets", "earliest") .option("failOnDataLoss", "false") - .option("fetchOffset.retryIntervalMs", "3000") + .option("fetchOffset.retryIntervalMs", "10000") val kafka = reader.load() .selectExpr("CAST(key AS STRING)", "CAST(value AS STRING)") .as[(String, String)] diff --git a/sql/hive/build.gradle b/sql/hive/build.gradle index 132ccbe66aa25..49900198d692a 100644 --- a/sql/hive/build.gradle +++ b/sql/hive/build.gradle @@ -57,7 +57,7 @@ dependencies { compile(group: 'org.apache.thrift', name: 'libthrift', version: thriftVersion) { exclude(group: 'org.slf4j', module: 'slf4j-api') } - compile(group: 'org.apache.thrift', name: 'libfb303', version: thriftVersion) { + compile(group: 'org.apache.thrift', name: 'libfb303', version: fb303Version) { exclude(group: 'org.slf4j', module: 'slf4j-api') } diff --git a/yarn/build.gradle b/yarn/build.gradle index 85fe7a2633c8a..2f7ceb5696d97 100644 --- a/yarn/build.gradle +++ b/yarn/build.gradle @@ -35,6 +35,7 @@ dependencies { exclude(group: 'org.ow2.asm', module: 'asm') exclude(group: 'org.jboss.netty', module: 'netty') exclude(group: 'javax.servlet', module: 'servlet-api') + exclude(group: 'org.apache.commons', module: 'commons-compress') exclude(group: 'commons-logging', module: 'commons-logging') exclude(group: 'com.sun.jersey') exclude(group: 'com.sun.jersey.jersey-test-framework') @@ -78,7 +79,7 @@ dependencies { compile(group: 'org.apache.thrift', name: 'libthrift', version: thriftVersion) { exclude(group: 'org.slf4j', module: 'slf4j-api') } - compile(group: 'org.apache.thrift', name: 'libfb303', version: thriftVersion) { + compile(group: 'org.apache.thrift', name: 'libfb303', version: fb303Version) { exclude(group: 'org.slf4j', module: 'slf4j-api') }