GraalVM native-image builds based on the Kafka 2.6.0 release

README.md

@@ -74,6 +74,7 @@ Very experimental.
 NOPUSH=true IMAGE_NAME=solsson/kafka:nativeagent ./hooks/build
 docker-compose -f native/docker-compose.yml down
 docker run --rm --entrypoint chown -v $(pwd)/native/configs:/configs busybox -R $(id -u) /configs
+# after reviewing config changes with git add -p native/configs
 NOPUSH=true IMAGE_NAME=solsson/kafka:native ./hooks/build
 ```
 

native/.dockerignore

@@ -0,0 +1,2 @@
+**/.*
+**/target

native/admincmd.Dockerfile

@@ -1,3 +1,9 @@
+FROM solsson/kafka:graalvm as substitutions
+
+WORKDIR /workspace
+COPY substitutions/admincmd .
+RUN mvn package
+
 FROM adoptopenjdk:11.0.8_10-jdk-hotspot-bionic@sha256:0513c0a82a82d1c9f4bfed18ef57bd5551ced2656342426a772c4772286dae1e \
   as nonlibs
 RUN echo "class Empty {public static void main(String[] a){}}" > Empty.java && javac Empty.java && jar --create --file /empty.jar Empty.class
@@ -7,12 +13,13 @@ FROM curlimages/curl@sha256:aa45e9d93122a3cfdf8d7de272e2798ea63733eeee6d06bd2ee4
 
 USER root
 RUN curl -sLS -o /slf4j-nop-1.7.30.jar https://repo1.maven.org/maven2/org/slf4j/slf4j-nop/1.7.30/slf4j-nop-1.7.30.jar
-RUN curl -sLS -o /quarkus-kafka-client-1.6.0.Final.jar https://repo1.maven.org/maven2/io/quarkus/quarkus-kafka-client/1.6.0.Final/quarkus-kafka-client-1.6.0.Final.jar
+RUN curl -sLS -o /quarkus-kafka-client-1.7.0.Final.jar https://repo1.maven.org/maven2/io/quarkus/quarkus-kafka-client/1.7.0.Final/quarkus-kafka-client-1.7.0.Final.jar
 
 FROM solsson/kafka:nativebase as native
 
 ARG classpath=/opt/kafka/libs/extensions/*:/opt/kafka/libs/*
 
+COPY --from=substitutions /workspace/target/*.jar /opt/kafka/libs/extensions/substitutions.jar
 COPY --from=extralibs /*.jar /opt/kafka/libs/extensions/
 
 # docker run --rm --entrypoint ls solsson/kafka -l /opt/kafka/libs/ | grep log
@@ -29,8 +36,7 @@ RUN native-image \
   -H:IncludeResourceBundles=joptsimple.ExceptionMessages \
   -H:ConfigurationFileDirectories=/home/nonroot/native-config \
   # When testing the build for a new version we should remove this one, but then it tends to come back
-  --allow-incomplete-classpath \
-  --report-unsupported-elements-at-runtime \
+  --initialize-at-build-time \
   # -D options from entrypoint
   -Djava.awt.headless=true \
   -Dkafka.logs.dir=/opt/kafka/bin/../logs \

native/configs-manual-additions/kafka-topics/reflect-config.json

@@ -2,9 +2,6 @@
   {
     "name": "org.apache.log4j.helpers.Loader"
   },
-  {
-    "name": "io.netty.util.internal.logging.Log4J2Logger"
-  },
   {
     "name": "sun.security.provider.ConfigFile",
     "methods": [

native/configs/kafka-configs/reflect-config.json

@@ -48,6 +48,9 @@
   "name":"java.lang.ThreadLocal",
   "methods":[{"name":"remove","parameterTypes":[] }]
 },
+{
+  "name":"java.lang.invoke.VarHandle"
+},
 {
   "name":"java.util.AbstractMap",
   "allDeclaredFields":true,
@@ -102,16 +105,10 @@
   "methods":[{"name":"<init>","parameterTypes":["org.apache.zookeeper.client.ZKClientConfig"] }]
 },
 {
-  "name":"scala.Serializable",
-  "allDeclaredMethods":true
-},
-{
-  "name":"scala.Tuple2[]"
-},
-{
-  "name":"scala.collection.convert.Wrappers$MapWrapper",
+  "name":"scala.collection.convert.JavaCollectionWrappers$MapWrapper",
   "allDeclaredFields":true,
-  "allDeclaredMethods":true
+  "allDeclaredMethods":true,
+  "allDeclaredConstructors":true
 },
 {
   "name":"sun.security.provider.ConfigFile",

native/configs/kafka-consumer-groups/reflect-config.json

@@ -11,6 +11,9 @@
   "name":"java.lang.Thread",
   "methods":[{"name":"getContextClassLoader","parameterTypes":[] }]
 },
+{
+  "name":"java.lang.invoke.VarHandle"
+},
 {
   "name":"kafka.admin.ConsumerGroupCommand$PartitionAssignmentState[]"
 },

native/configs/kafka-server-start/reflect-config.json

@@ -35,9 +35,6 @@
   "allDeclaredFields":true,
   "allDeclaredMethods":true
 },
-{
-  "name":"java.lang.Runnable[]"
-},
 {
   "name":"java.lang.Runtime",
   "methods":[{"name":"version","parameterTypes":[] }]
@@ -61,6 +58,9 @@
   "name":"java.lang.ThreadLocal",
   "methods":[{"name":"remove","parameterTypes":[] }]
 },
+{
+  "name":"java.lang.invoke.VarHandle"
+},
 {
   "name":"java.util.AbstractCollection",
   "allDeclaredFields":true,
@@ -91,16 +91,10 @@
 {
   "name":"java.util.zip.CRC32C"
 },
-{
-  "name":"kafka.server.DelayedOperationPurgatory$WatcherList[]"
-},
 {
   "name":"kafka.utils.Log4jController",
   "methods":[{"name":"<init>","parameterTypes":[] }]
 },
-{
-  "name":"kafka.utils.timer.TimerTaskList[]"
-},
 {
   "name":"org.apache.kafka.common.protocol.types.Struct[]"
 },
@@ -150,35 +144,32 @@
   "allDeclaredMethods":true
 },
 {
-  "name":"scala.Serializable",
-  "allDeclaredMethods":true
-},
-{
-  "name":"scala.Tuple2[]"
-},
-{
-  "name":"scala.collection.convert.Wrappers$IterableWrapperTrait",
+  "name":"scala.collection.convert.JavaCollectionWrappers$IterableWrapperTrait",
   "allDeclaredMethods":true
 },
 {
-  "name":"scala.collection.convert.Wrappers$MapWrapper",
+  "name":"scala.collection.convert.JavaCollectionWrappers$MapWrapper",
   "allDeclaredFields":true,
-  "allDeclaredMethods":true
+  "allDeclaredMethods":true,
+  "allDeclaredConstructors":true
 },
 {
-  "name":"scala.collection.convert.Wrappers$MutableBufferWrapper",
+  "name":"scala.collection.convert.JavaCollectionWrappers$MutableBufferWrapper",
   "allDeclaredFields":true,
-  "allDeclaredMethods":true
+  "allDeclaredMethods":true,
+  "allDeclaredConstructors":true
 },
 {
-  "name":"scala.collection.convert.Wrappers$MutableMapWrapper",
+  "name":"scala.collection.convert.JavaCollectionWrappers$MutableMapWrapper",
   "allDeclaredFields":true,
-  "allDeclaredMethods":true
+  "allDeclaredMethods":true,
+  "allDeclaredConstructors":true
 },
 {
-  "name":"scala.collection.convert.Wrappers$SeqWrapper",
+  "name":"scala.collection.convert.JavaCollectionWrappers$SeqWrapper",
   "allDeclaredFields":true,
-  "allDeclaredMethods":true
+  "allDeclaredMethods":true,
+  "allDeclaredConstructors":true
 },
 {
   "name":"sun.misc.Signal",

native/configs/kafka-topics/reflect-config.json

@@ -3,9 +3,6 @@
   "name":"com.fasterxml.jackson.databind.ext.Java7SupportImpl",
   "methods":[{"name":"<init>","parameterTypes":[] }]
 },
-{
-  "name":"io.netty.util.internal.logging.Log4J2Logger"
-},
 {
   "name":"java.lang.Integer",
   "methods":[{"name":"valueOf","parameterTypes":["java.lang.String"] }]
@@ -33,6 +30,9 @@
   "name":"java.lang.ThreadLocal",
   "methods":[{"name":"remove","parameterTypes":[] }]
 },
+{
+  "name":"java.lang.invoke.VarHandle"
+},
 {
   "name":"kafka.utils.Log4jController",
   "methods":[{"name":"<init>","parameterTypes":[] }]
@@ -74,9 +74,6 @@
   "name":"org.apache.zookeeper.ClientCnxnSocketNIO",
   "methods":[{"name":"<init>","parameterTypes":["org.apache.zookeeper.client.ZKClientConfig"] }]
 },
-{
-  "name":"scala.Tuple2[]"
-},
 {
   "name":"sun.security.provider.ConfigFile",
   "methods":[{"name":"<init>","parameterTypes":[] }]

native/configs/zookeeper-server-start/reflect-config.json

@@ -25,26 +25,18 @@
   "name":"org.apache.log4j.Category",
   "methods":[{"name":"getName","parameterTypes":[] }]
 },
-{
-  "name":"org.apache.log4j.CategoryKey"
-},
 {
   "name":"org.apache.log4j.ConsoleAppender",
   "methods":[{"name":"<init>","parameterTypes":[] }]
 },
-{
-  "name":"org.apache.log4j.Hierarchy",
-  "methods":[{"name":"getCurrentLoggers","parameterTypes":[] }]
-},
 {
   "name":"org.apache.log4j.Layout"
 },
 {
   "name":"org.apache.log4j.Level"
 },
 {
-  "name":"org.apache.log4j.LogManager",
-  "methods":[{"name":"getLoggerRepository","parameterTypes":[] }]
+  "name":"org.apache.log4j.LogManager"
 },
 {
   "name":"org.apache.log4j.Logger",
@@ -57,23 +49,6 @@
 {
   "name":"org.apache.log4j.Priority"
 },
-{
-  "name":"org.apache.log4j.jmx.AppenderDynamicMBean",
-  "allPublicConstructors":true
-},
-{
-  "name":"org.apache.log4j.jmx.HierarchyDynamicMBean",
-  "allPublicConstructors":true,
-  "methods":[{"name":"addLoggerMBean","parameterTypes":["java.lang.String"] }]
-},
-{
-  "name":"org.apache.log4j.jmx.LayoutDynamicMBean",
-  "allPublicConstructors":true
-},
-{
-  "name":"org.apache.log4j.jmx.LoggerDynamicMBean",
-  "allPublicConstructors":true
-},
 {
   "name":"org.apache.log4j.spi.OptionHandler"
 },

native/docker-compose.yml

@@ -133,7 +133,8 @@
         - --group=testgroup
         - --topic=topic1
         - --reset-offsets
-        - --to-offset 0
+        - --to-offset=0
+        #- --execute intentionally left out, for the dry-run use case
 
       step7:
         image: solsson/kafka:nativeagent-kafka-consumer-groups
@@ -165,6 +166,12 @@
         -   --add-config
         -     max.message.bytes=2048
 
+      step10:
+        image: solsson/kafka:nativeagent-kafka-consumer-groups
+        volumes: [             ./configs/kafka-consumer-groups:/home/nonroot/native-config ]
+        command:
+        - --intentionally wrong
+
       kafkacat:
         image: edenhill/kafkacat:1.5.0
         depends_on:

native/native-usecases.sh

@@ -33,6 +33,6 @@ $compose up -d kafka-0
 $compose up -d kafkacat
 $compose ps
 
-for step in $(seq 1 9); do
+for step in $(seq 1 10); do
   $compose up step$step
 done

native/substitutions/admincmd/.gitignore

@@ -0,0 +1,4 @@
+.classpath
+.project
+.settings
+target