Skip to content

Commit

Permalink
Fix Hadoop Indexing CLI for versions JDK v9+
Browse files Browse the repository at this point in the history
For JDK v9+ this job will fail as URLClassloader is no longer the default classloader. Additionally, with the introduction of the jmod system, the job failed to find the correct JDK-included modules because the new classloader constructed for the job didn't reference the system classloader. Adding ClassLoader.getSystemClassLoader() as the parent classloader fixes this.
  • Loading branch information
jtuglu-netflix committed Dec 16, 2024
1 parent 12eed75 commit dec1188
Showing 1 changed file with 9 additions and 6 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import org.apache.druid.indexing.common.config.TaskConfig;
import org.apache.druid.indexing.common.task.Initialization;
import org.apache.druid.java.util.common.logger.Logger;
import org.apache.druid.utils.JvmUtils;

import java.io.File;
import java.lang.reflect.Method;
Expand All @@ -39,6 +40,7 @@
import java.util.List;

/**
*
*/
@Command(
name = "hadoop",
Expand All @@ -55,12 +57,12 @@ public class CliHadoopIndexer implements Runnable
private String argumentSpec;

@Option(name = {"-c", "--coordinate", "hadoopDependencies"},
description = "extra dependencies to pull down (e.g. non-default hadoop coordinates or extra hadoop jars)")
description = "extra dependencies to pull down (e.g. non-default hadoop coordinates or extra hadoop jars)")
@SuppressWarnings("MismatchedQueryAndUpdateOfCollection")
private List<String> coordinates;

@Option(name = "--no-default-hadoop",
description = "don't pull down the default hadoop version")
description = "don't pull down the default hadoop version")
public boolean noDefaultHadoop;

@Inject
Expand All @@ -84,9 +86,7 @@ public void run()
extensionURLs.addAll(Arrays.asList(extensionLoader.getURLs()));
}

final List<URL> nonHadoopURLs = Arrays.asList(
((URLClassLoader) CliHadoopIndexer.class.getClassLoader()).getURLs()
);
final List<URL> nonHadoopURLs = JvmUtils.systemClassPath();

final List<URL> driverURLs = new ArrayList<>(nonHadoopURLs);
// put hadoop dependencies last to avoid jets3t & apache.httpcore version conflicts
Expand All @@ -95,7 +95,10 @@ public void run()
driverURLs.addAll(Arrays.asList(hadoopLoader.getURLs()));
}

final URLClassLoader loader = new URLClassLoader(driverURLs.toArray(new URL[0]), null);
final URLClassLoader loader = new URLClassLoader(
driverURLs.toArray(new URL[0]),
ClassLoader.getSystemClassLoader()
);
Thread.currentThread().setContextClassLoader(loader);

final List<URL> jobUrls = new ArrayList<>();
Expand Down

0 comments on commit dec1188

Please sign in to comment.