Skip to content

Commit

Permalink
spark: load config resolver
Browse files Browse the repository at this point in the history
  • Loading branch information
jzhuge committed Jul 30, 2024
1 parent 355ceab commit 68c8949
Showing 1 changed file with 30 additions and 3 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@
import com.netflix.spectator.sidecar.SidecarRegistry;
import com.typesafe.config.Config;
import com.typesafe.config.ConfigFactory;
import com.typesafe.config.ConfigResolveOptions;
import com.typesafe.config.ConfigResolver;
import org.apache.spark.metrics.sink.Sink;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand Down Expand Up @@ -56,7 +58,7 @@ public SparkSink(
Properties properties,
MetricRegistry registry,
org.apache.spark.SecurityManager manager) {
final Config config = loadConfig();
final Config config = loadConfig(properties);
sidecarRegistry = new SidecarRegistry(
Clock.SYSTEM, new SpectatorConfig(config.getConfig("spectator.spark.sidecar")));
reporter = SpectatorReporter.forRegistry(registry)
Expand All @@ -76,8 +78,33 @@ public SparkSink(
}
}

private Config loadConfig() {
return ConfigFactory.load(pickClassLoader());
private Config loadConfig(Properties properties) {
ClassLoader classLoader = pickClassLoader();
ConfigResolver configResolver = getConfigResolver(properties);
if (configResolver != null) {
ConfigResolveOptions resolveOptions = ConfigResolveOptions.defaults().appendResolver(configResolver);
return ConfigFactory.load(classLoader, resolveOptions);
} else {
return ConfigFactory.load(classLoader);
}
}

private static ConfigResolver getConfigResolver(Properties properties) {
final String resolverClassName = properties.getProperty("resolver.class");
if (resolverClassName == null) {
return null;
}
try {
Class<?> clazz = Class.forName(resolverClassName);
ConfigResolver resolver = (ConfigResolver) clazz.getDeclaredConstructor().newInstance();
LOGGER.info("Created ConfigResolver instance for {}", resolverClassName);
return resolver;
} catch (ClassNotFoundException e) {
LOGGER.error("Class {} not found", resolverClassName, e);
} catch (Exception e) {
LOGGER.error("Error creating instance for {}", resolverClassName, e);
}
return null;
}

@SuppressWarnings("PMD.UseProperClassLoader")
Expand Down

0 comments on commit 68c8949

Please sign in to comment.