[SPARK-53849][BUILD] Upgrade Netty to 4.2.6.Final

### What changes were proposed in this pull request?

This PR aims to upgrade `Netty` to `4.2.6.Final` as a part of `Apache Spark 4.1.0` preparation.

This PR follows the official community migration guide.
- https://github.com/netty/netty/wiki/Netty-4.2-Migration-Guide

### Why are the changes needed?

`Netty` community recommends `4.2`.

> After a lot of work and many contributions from our OSS community we are thrilled to finally announce the release of netty 4.2.0.Final! Everyone using netty 4.1.x should be able to upgrade to 4.2.0.Final without any API breakage. The only new requirement is JDK8 or later (and JDK9+ for io_uring) and a more recent GLIBC version when using our native transports.

- https://netty.io/news/2025/09/08/4-2-6.html
- https://netty.io/news/2025/09/03/4-2-5.html
- https://netty.io/news/2025/08/13/4-2-4.html
- https://netty.io/news/2025/07/15/4-2-3.html
- https://netty.io/news/2025/06/05/4-2-2.html
- https://netty.io/news/2025/05/06/4-2-1.html
- https://netty.io/news/2025/04/03/4-2-0.html

### Does this PR introduce _any_ user-facing change?

No behavior change.

### How was this patch tested?

Pass the CIs.

### Was this patch authored or co-authored using generative AI tooling?

No.

Closes #52552 from dongjoon-hyun/SPARK-53849.

Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>

Author: dongjoon-hyun

common/network-common/src/test/java/org/apache/spark/network/client/TransportClientFactorySuite.java

@@ -225,11 +225,12 @@ public void closeFactoryBeforeCreateClient() {
   }
 
   @Test
-  public void fastFailConnectionInTimeWindow() {
+  public void fastFailConnectionInTimeWindow() throws IOException, InterruptedException {
     TransportClientFactory factory = context.createClientFactory();
     TransportServer server = context.createServer();
     int unreachablePort = server.getPort();
     server.close();
+    Thread.sleep(1000);
     Assertions.assertThrows(IOException.class,
       () -> factory.createClient(TestUtils.getLocalHost(), unreachablePort, true));
     Assertions.assertThrows(IOException.class,
@@ -258,6 +259,7 @@ public void unlimitedConnectionAndCreationTimeouts() throws IOException, Interru
       TransportServer server = ctx.createServer();
       int unreachablePort = server.getPort();
       JavaUtils.closeQuietly(server);
+      Thread.sleep(1000);
       IOException exception = Assertions.assertThrows(IOException.class,
           () -> factory.createClient(TestUtils.getLocalHost(), unreachablePort, true));
       assertNotEquals(exception.getCause(), null);

dev/deps/spark-deps-hadoop-3-hive-2.3

@@ -198,33 +198,48 @@ metrics-jmx/4.2.33//metrics-jmx-4.2.33.jar
 metrics-json/4.2.33//metrics-json-4.2.33.jar
 metrics-jvm/4.2.33//metrics-jvm-4.2.33.jar
 minlog/1.3.0//minlog-1.3.0.jar
-netty-all/4.1.127.Final//netty-all-4.1.127.Final.jar
-netty-buffer/4.1.127.Final//netty-buffer-4.1.127.Final.jar
-netty-codec-dns/4.1.127.Final//netty-codec-dns-4.1.127.Final.jar
-netty-codec-http/4.1.127.Final//netty-codec-http-4.1.127.Final.jar
-netty-codec-http2/4.1.127.Final//netty-codec-http2-4.1.127.Final.jar
-netty-codec-socks/4.1.127.Final//netty-codec-socks-4.1.127.Final.jar
-netty-codec/4.1.127.Final//netty-codec-4.1.127.Final.jar
-netty-common/4.1.127.Final//netty-common-4.1.127.Final.jar
-netty-handler-proxy/4.1.127.Final//netty-handler-proxy-4.1.127.Final.jar
-netty-handler/4.1.127.Final//netty-handler-4.1.127.Final.jar
-netty-resolver-dns/4.1.127.Final//netty-resolver-dns-4.1.127.Final.jar
-netty-resolver/4.1.127.Final//netty-resolver-4.1.127.Final.jar
+netty-all/4.2.6.Final//netty-all-4.2.6.Final.jar
+netty-buffer/4.2.6.Final//netty-buffer-4.2.6.Final.jar
+netty-codec-base/4.2.6.Final//netty-codec-base-4.2.6.Final.jar
+netty-codec-classes-quic/4.2.6.Final//netty-codec-classes-quic-4.2.6.Final.jar
+netty-codec-compression/4.2.6.Final//netty-codec-compression-4.2.6.Final.jar
+netty-codec-dns/4.2.6.Final//netty-codec-dns-4.2.6.Final.jar
+netty-codec-http/4.2.6.Final//netty-codec-http-4.2.6.Final.jar
+netty-codec-http2/4.2.6.Final//netty-codec-http2-4.2.6.Final.jar
+netty-codec-http3/4.2.6.Final//netty-codec-http3-4.2.6.Final.jar
+netty-codec-marshalling/4.2.6.Final//netty-codec-marshalling-4.2.6.Final.jar
+netty-codec-native-quic/4.2.6.Final/linux-aarch_64/netty-codec-native-quic-4.2.6.Final-linux-aarch_64.jar
+netty-codec-native-quic/4.2.6.Final/linux-x86_64/netty-codec-native-quic-4.2.6.Final-linux-x86_64.jar
+netty-codec-native-quic/4.2.6.Final/osx-aarch_64/netty-codec-native-quic-4.2.6.Final-osx-aarch_64.jar
+netty-codec-native-quic/4.2.6.Final/osx-x86_64/netty-codec-native-quic-4.2.6.Final-osx-x86_64.jar
+netty-codec-native-quic/4.2.6.Final/windows-x86_64/netty-codec-native-quic-4.2.6.Final-windows-x86_64.jar
+netty-codec-protobuf/4.2.6.Final//netty-codec-protobuf-4.2.6.Final.jar
+netty-codec-socks/4.2.6.Final//netty-codec-socks-4.2.6.Final.jar
+netty-codec/4.2.6.Final//netty-codec-4.2.6.Final.jar
+netty-common/4.2.6.Final//netty-common-4.2.6.Final.jar
+netty-handler-proxy/4.2.6.Final//netty-handler-proxy-4.2.6.Final.jar
+netty-handler/4.2.6.Final//netty-handler-4.2.6.Final.jar
+netty-resolver-dns/4.2.6.Final//netty-resolver-dns-4.2.6.Final.jar
+netty-resolver/4.2.6.Final//netty-resolver-4.2.6.Final.jar
 netty-tcnative-boringssl-static/2.0.74.Final/linux-aarch_64/netty-tcnative-boringssl-static-2.0.74.Final-linux-aarch_64.jar
 netty-tcnative-boringssl-static/2.0.74.Final/linux-x86_64/netty-tcnative-boringssl-static-2.0.74.Final-linux-x86_64.jar
 netty-tcnative-boringssl-static/2.0.74.Final/osx-aarch_64/netty-tcnative-boringssl-static-2.0.74.Final-osx-aarch_64.jar
 netty-tcnative-boringssl-static/2.0.74.Final/osx-x86_64/netty-tcnative-boringssl-static-2.0.74.Final-osx-x86_64.jar
 netty-tcnative-boringssl-static/2.0.74.Final/windows-x86_64/netty-tcnative-boringssl-static-2.0.74.Final-windows-x86_64.jar
 netty-tcnative-classes/2.0.74.Final//netty-tcnative-classes-2.0.74.Final.jar
-netty-transport-classes-epoll/4.1.127.Final//netty-transport-classes-epoll-4.1.127.Final.jar
-netty-transport-classes-kqueue/4.1.127.Final//netty-transport-classes-kqueue-4.1.127.Final.jar
-netty-transport-native-epoll/4.1.127.Final/linux-aarch_64/netty-transport-native-epoll-4.1.127.Final-linux-aarch_64.jar
-netty-transport-native-epoll/4.1.127.Final/linux-riscv64/netty-transport-native-epoll-4.1.127.Final-linux-riscv64.jar
-netty-transport-native-epoll/4.1.127.Final/linux-x86_64/netty-transport-native-epoll-4.1.127.Final-linux-x86_64.jar
-netty-transport-native-kqueue/4.1.127.Final/osx-aarch_64/netty-transport-native-kqueue-4.1.127.Final-osx-aarch_64.jar
-netty-transport-native-kqueue/4.1.127.Final/osx-x86_64/netty-transport-native-kqueue-4.1.127.Final-osx-x86_64.jar
-netty-transport-native-unix-common/4.1.127.Final//netty-transport-native-unix-common-4.1.127.Final.jar
-netty-transport/4.1.127.Final//netty-transport-4.1.127.Final.jar
+netty-transport-classes-epoll/4.2.6.Final//netty-transport-classes-epoll-4.2.6.Final.jar
+netty-transport-classes-io_uring/4.2.6.Final//netty-transport-classes-io_uring-4.2.6.Final.jar
+netty-transport-classes-kqueue/4.2.6.Final//netty-transport-classes-kqueue-4.2.6.Final.jar
+netty-transport-native-epoll/4.2.6.Final/linux-aarch_64/netty-transport-native-epoll-4.2.6.Final-linux-aarch_64.jar
+netty-transport-native-epoll/4.2.6.Final/linux-riscv64/netty-transport-native-epoll-4.2.6.Final-linux-riscv64.jar
+netty-transport-native-epoll/4.2.6.Final/linux-x86_64/netty-transport-native-epoll-4.2.6.Final-linux-x86_64.jar
+netty-transport-native-io_uring/4.2.6.Final/linux-aarch_64/netty-transport-native-io_uring-4.2.6.Final-linux-aarch_64.jar
+netty-transport-native-io_uring/4.2.6.Final/linux-riscv64/netty-transport-native-io_uring-4.2.6.Final-linux-riscv64.jar
+netty-transport-native-io_uring/4.2.6.Final/linux-x86_64/netty-transport-native-io_uring-4.2.6.Final-linux-x86_64.jar
+netty-transport-native-kqueue/4.2.6.Final/osx-aarch_64/netty-transport-native-kqueue-4.2.6.Final-osx-aarch_64.jar
+netty-transport-native-kqueue/4.2.6.Final/osx-x86_64/netty-transport-native-kqueue-4.2.6.Final-osx-x86_64.jar
+netty-transport-native-unix-common/4.2.6.Final//netty-transport-native-unix-common-4.2.6.Final.jar
+netty-transport/4.2.6.Final//netty-transport-4.2.6.Final.jar
 objenesis/3.4//objenesis-3.4.jar
 okhttp/3.12.12//okhttp-3.12.12.jar
 okio/1.17.6//okio-1.17.6.jar

launcher/src/main/java/org/apache/spark/launcher/JavaModuleOptions.java

@@ -45,6 +45,8 @@ public class JavaModuleOptions {
       "--add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED",
       "-Djdk.reflect.useDirectMethodHandle=false",
       "-Dio.netty.tryReflectionSetAccessible=true",
+      "-Dio.netty.allocator.type=pooled",
+      "-Dio.netty.handler.ssl.defaultEndpointVerificationAlgorithm=NONE",
       "--enable-native-access=ALL-UNNAMED"};
 
     /**

pom.xml

@@ -217,7 +217,7 @@
     <bouncycastle.version>1.82</bouncycastle.version>
     <tink.version>1.16.0</tink.version>
     <datasketches.version>6.2.0</datasketches.version>
-    <netty.version>4.1.127.Final</netty.version>
+    <netty.version>4.2.6.Final</netty.version>
     <netty-tcnative.version>2.0.74.Final</netty-tcnative.version>
     <icu4j.version>77.1</icu4j.version>
     <junit-jupiter.version>5.13.4</junit-jupiter.version>
@@ -337,6 +337,8 @@
       --add-opens=java.base/sun.util.calendar=ALL-UNNAMED
       -Djdk.reflect.useDirectMethodHandle=false
       -Dio.netty.tryReflectionSetAccessible=true
+      -Dio.netty.allocator.type=pooled
+      -Dio.netty.handler.ssl.defaultEndpointVerificationAlgorithm=NONE
       --enable-native-access=ALL-UNNAMED
     </extraJavaTestArgs>
     <mariadb.java.client.version>2.7.12</mariadb.java.client.version>

project/SparkBuild.scala

@@ -1704,6 +1704,8 @@ object TestSettings {
         "--add-opens=java.base/sun.util.calendar=ALL-UNNAMED",
         "-Djdk.reflect.useDirectMethodHandle=false",
         "-Dio.netty.tryReflectionSetAccessible=true",
+        "-Dio.netty.allocator.type=pooled",
+        "-Dio.netty.handler.ssl.defaultEndpointVerificationAlgorithm=NONE",
         "--enable-native-access=ALL-UNNAMED").mkString(" ")
       s"-Xmx$heapSize -Xss4m -XX:MaxMetaspaceSize=$metaspaceSize -XX:ReservedCodeCacheSize=128m -Dfile.encoding=UTF-8 $extraTestJavaArgs"
         .split(" ").toSeq