diff --git a/README.md b/README.md index 9bfe8f62..7612d7cf 100644 --- a/README.md +++ b/README.md @@ -216,3 +216,22 @@ Configuring radar-upload-frontend requires configuring the right environment var + +## Sentry monitoring + +To enable Sentry monitoring for the kafka-connect-upload-source and radar-upload-backend services: + +1. Set a `SENTRY_DSN` environment variable that points to the desired Sentry DSN. +2. (Optional) Set the `SENTRY_LOG_LEVEL` environment variable to control the minimum log level of + events sent to Sentry. + The default log level for Sentry is `WARN`. Possible values are `TRACE`, `DEBUG`, `INFO`, `WARN`, + and `ERROR`. + +For further configuration of Sentry via environmental variables see [here](https://docs.sentry.io/platforms/java/configuration/#configuration-via-the-runtime-environment). For instance: + +``` +SENTRY_LOG_LEVEL: 'ERROR' +SENTRY_DSN: 'https://000000000000.ingest.de.sentry.io/000000000000' +SENTRY_ATTACHSTACKTRACE: true +SENTRY_STACKTRACE_APP_PACKAGES: io.confluent.connect,org.radarbase.connect.rest +``` diff --git a/build.gradle.kts b/build.gradle.kts index f64f3c12..30a3136c 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,14 +1,7 @@ -import org.radarbase.gradle.plugin.radarKotlin - plugins { id("org.radarbase.radar-root-project") version Versions.radarCommons id("org.radarbase.radar-dependency-management") version Versions.radarCommons id("org.radarbase.radar-kotlin") version Versions.radarCommons apply false - - id("org.jetbrains.kotlin.plugin.noarg") version Versions.kotlin apply false - id("org.jetbrains.kotlin.plugin.jpa") version Versions.kotlin apply false - id("org.jetbrains.kotlin.plugin.allopen") version Versions.kotlin apply false - id("com.avast.gradle.docker-compose") version Versions.dockerCompose apply false } allprojects { @@ -16,27 +9,7 @@ allprojects { version = "0.5.14" } - radarRootProject { projectVersion.set(Versions.project) gradleVersion.set(Versions.wrapper) } - - -subprojects { - apply(plugin = "org.radarbase.radar-kotlin") - - radarKotlin { - javaVersion.set(Versions.java) - kotlinVersion.set(Versions.kotlin) - slf4jVersion.set(Versions.slf4j) - log4j2Version.set(Versions.log4j2) - junitVersion.set(Versions.junit) - } -} - -project(":kafka-connect-upload-source") { - radarKotlin { - javaVersion.set(17) - } -} diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts index 8a695a47..7fe21d8a 100644 --- a/buildSrc/build.gradle.kts +++ b/buildSrc/build.gradle.kts @@ -1,5 +1,5 @@ plugins { - kotlin("jvm") version "1.9.10" + id("org.radarbase.radar-kotlin") version "1.1.3" } repositories { diff --git a/buildSrc/src/main/kotlin/Versions.kt b/buildSrc/src/main/kotlin/Versions.kt index b462d3b2..206148cc 100644 --- a/buildSrc/src/main/kotlin/Versions.kt +++ b/buildSrc/src/main/kotlin/Versions.kt @@ -7,7 +7,7 @@ object Versions { const val wrapper = "8.4" const val dockerCompose = "0.17.5" - const val radarCommons = "1.1.1" + const val radarCommons = "1.1.3" const val confluent = "7.5.1" const val kafka = "$confluent-ce" @@ -16,7 +16,7 @@ object Versions { const val ktor = "2.3.5" const val log4j2 = "2.21.1" - const val slf4j = "2.0.9" + const val sentryLog4j = "1.7.30" const val okhttp = "4.12.0" diff --git a/docker-compose.yml b/docker-compose.yml index 90bdd567..3075e516 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -182,7 +182,7 @@ services: build: context: . dockerfile: kafka-connect-upload-source/Dockerfile - image: radarbase/radar-connect-upload-source:0.5.1 + image: radarbase/radar-connect-upload-source:latest restart: on-failure volumes: - ./etc/source-upload.properties:/etc/kafka-connect/source-upload.properties @@ -194,7 +194,7 @@ services: - kafka-2 - kafka-3 - schema-registry-1 - - radar-upload-backend + #- radar-upload-backend environment: CONNECT_BOOTSTRAP_SERVERS: PLAINTEXT://kafka-1:9092,PLAINTEXT://kafka-2:9092,PLAINTEXT://kafka-3:9092 CONNECT_REST_PORT: 8083 @@ -216,31 +216,37 @@ services: KAFKA_HEAP_OPTS: "-Xms256m -Xmx768m" KAFKA_BROKERS: 3 CONNECT_LOG4J_LOGGERS: "org.reflections=ERROR" + # SENTRY_LOG_LEVEL: 'ERROR' + # SENTRY_DSN: 'https://000000000000.ingest.de.sentry.io/000000000000' + # SENTRY_ATTACHSTACKTRACE: true + # SENTRY_STACKTRACE_APP_PACKAGES: io.confluent.connect,org.radarbase.connect.rest - radar-upload-backend: - build: - context: . - dockerfile: radar-upload-backend/Dockerfile - image: radarbase/radar-upload-connect-backend:0.5.1 - ports: - - "8085:8085" - depends_on: - - managementportal-app - - radarbase-postgresql - volumes: - - ./etc/upload.yml:/etc/upload-backend/upload.yml - - ./etc/logback.xml:/etc/upload-backend/logback.xml - environment: - JAVA_OPTS: "-Dlogback.configurationFile=/etc/upload-backend/logback.xml" - command: ["radar-upload-backend", "/etc/upload-backend/upload.yml"] +# radar-upload-backend: +# build: +# context: . +# dockerfile: radar-upload-backend/Dockerfile +# image: radarbase/radar-upload-connect-backend:latest +# ports: +# - "8085:8085" +# depends_on: +# - managementportal-app +# - radarbase-postgresql +# volumes: +# - ./etc/upload.yml:/etc/upload-backend/upload.yml +# command: ["radar-upload-backend", "/etc/upload-backend/upload.yml"] + # environment: + # SENTRY_LOG_LEVEL: 'ERROR' + # SENTRY_DSN: 'https://000000000000.ingest.de.sentry.io/000000000000' + # SENTRY_ATTACHSTACKTRACE: true + # SENTRY_STACKTRACE_APP_PACKAGES: io.confluent.connect,org.radarbase.upload radar-upload-frontend: build: context: radar-upload-frontend dockerfile: Dockerfile - image: radarbase/radar-upload-connect-frontend:0.5.1 + image: radarbase/radar-upload-connect-frontend:latest depends_on: - - radar-upload-backend + #- radar-upload-backend - managementportal-app environment: VUE_APP_BASE_URL: "/upload" @@ -260,7 +266,7 @@ services: ports: - "8080:8080" depends_on: - - radar-upload-backend + #- radar-upload-backend - managementportal-app - radar-upload-frontend volumes: diff --git a/etc/logback.xml b/etc/logback.xml deleted file mode 100644 index 059befbd..00000000 --- a/etc/logback.xml +++ /dev/null @@ -1,17 +0,0 @@ - - - - - - %date{yyyy-MM-dd HH:mm:ss,UTC} UTC [%.2thread] %-5level - %msg [%logger:%line] %n - - - - - - - - - - - diff --git a/integration-test/build.gradle.kts b/integration-test/build.gradle.kts index a8737a1a..3a2ac699 100644 --- a/integration-test/build.gradle.kts +++ b/integration-test/build.gradle.kts @@ -2,8 +2,8 @@ import org.jetbrains.kotlin.cli.common.toBooleanLenient import java.time.Duration plugins { - kotlin("jvm") - id("com.avast.gradle.docker-compose") + id("com.avast.gradle.docker-compose") version Versions.dockerCompose + id("org.radarbase.radar-kotlin") } sourceSets { @@ -16,8 +16,6 @@ sourceSets { } dependencies { - implementation(kotlin("stdlib-jdk8")) - testImplementation("io.confluent:kafka-connect-avro-converter:${Versions.confluent}") testImplementation(platform("com.fasterxml.jackson:jackson-bom:${Versions.jackson}")) testImplementation("com.squareup.okhttp3:okhttp:${Versions.okhttp}") @@ -50,3 +48,8 @@ dockerCompose { environment.put("SERVICES_HOST", "localhost") isRequiredBy(tasks["integrationTest"]) } + +radarKotlin { + // TODO remove after using new release of radar-kotlin plugin + javaVersion.set(Versions.java) +} diff --git a/kafka-connect-upload-source/Dockerfile b/kafka-connect-upload-source/Dockerfile index 0ca3d18e..1a7a8b39 100644 --- a/kafka-connect-upload-source/Dockerfile +++ b/kafka-connect-upload-source/Dockerfile @@ -17,7 +17,7 @@ RUN gradle jar FROM confluentinc/cp-kafka-connect-base:7.7.1 -MAINTAINER @nivemaham @blootsvoets +MAINTAINER @pvannierop LABEL description="Kafka Data Upload Source connector" @@ -28,11 +28,16 @@ COPY --from=builder /code/kafka-connect-upload-source/build/third-party/*.jar ${ COPY --from=builder /code/kafka-connect-upload-source/build/libs/kafka-connect-upload-source-*.jar ${CONNECT_PLUGIN_PATH}/kafka-connect-upload-source/ # Load topics validator -COPY ./kafka-connect-upload-source/src/main/docker/kafka-wait /usr/bin/kafka-wait +COPY ./kafka-connect-upload-source/src/main/docker/ensure /etc/confluent/docker/ensure # Load modified launcher COPY ./kafka-connect-upload-source/src/main/docker/launch /etc/confluent/docker/launch +# Overwrite the log4j configuration to include Sentry monitoring. +COPY ./kafka-connect-upload-source/src/main/docker/log4j.properties.template /etc/confluent/docker/log4j.properties.template +# Copy Sentry monitoring jars. +COPY --from=builder /code/kafka-connect-upload-source/build/third-party/sentry-* /etc/kafka-connect/jars + USER "root" # create parent directory for storing offsets in standalone mode RUN mkdir -p /var/lib/kafka-connect-upload-source/logs \ diff --git a/kafka-connect-upload-source/build.gradle.kts b/kafka-connect-upload-source/build.gradle.kts index 969a1353..b295e238 100644 --- a/kafka-connect-upload-source/build.gradle.kts +++ b/kafka-connect-upload-source/build.gradle.kts @@ -1,6 +1,5 @@ plugins { - java - kotlin("jvm") + id("org.radarbase.radar-kotlin") } sourceSets { @@ -32,7 +31,6 @@ dependencies { // Included in connector runtime compileOnly("org.apache.kafka:connect-api:${Versions.kafka}") - implementation(kotlin("stdlib-jdk8")) implementation(kotlin("reflect")) testImplementation("io.confluent:kafka-connect-avro-converter:${Versions.confluent}") @@ -40,6 +38,14 @@ dependencies { testImplementation("org.hamcrest:hamcrest:${Versions.hamcrest}") testImplementation("org.apache.kafka:connect-api:${Versions.kafka}") testImplementation("org.mockito.kotlin:mockito-kotlin:${Versions.mockitoKotlin}") + + // Application monitoring + // This dependency is not used by the upload connector, but copied into the Docker image (Dockerfile) + compileOnly("io.sentry:sentry-log4j:${Versions.sentryLog4j}") { + // Exclude log4j with security vulnerability (safe version is provided by docker image). + exclude(group = "log4j", module = "log4j") + } + } task("integrationTest") { @@ -49,3 +55,7 @@ task("integrationTest") { classpath = sourceSets["integrationTest"].runtimeClasspath mustRunAfter(tasks["test"]) } + +radarKotlin { + javaVersion.set(Versions.java) +} diff --git a/kafka-connect-upload-source/src/main/docker/ensure b/kafka-connect-upload-source/src/main/docker/ensure new file mode 100755 index 00000000..13e94dad --- /dev/null +++ b/kafka-connect-upload-source/src/main/docker/ensure @@ -0,0 +1,88 @@ +#!/bin/bash + +if [ "$WAIT_FOR_KAFKA" != "1" ]; then + echo "Starting without checking for Kafka availability" + exit 0 +fi + +max_timeout=32 + +IS_TEMP=0 + +echo "===> Wait for infrastructure ..." + +if [ -z "$COMMAND_CONFIG_FILE_PATH" ]; then + COMMAND_CONFIG_FILE_PATH="$(mktemp)" + IS_TEMP=1 +fi + +if [ ! -f "$COMMAND_CONFIG_FILE_PATH" ] || [ $IS_TEMP = 1 ]; then + while IFS='=' read -r -d '' n v; do + if [[ "$n" == "CONNECT_"* ]]; then + name="${n/CONNECT_/""}" # remove first "CONNECT_" + name="${name,,}" # lower case + name="${name//_/"."}" # replace all '_' with '.' + echo "$name=$v" >> ${COMMAND_CONFIG_FILE_PATH} + fi + done < <(env -0) +fi + +# Check if variables exist +if [ -z "$CONNECT_BOOTSTRAP_SERVERS" ]; then + echo "CONNECT_BOOTSTRAP_SERVERS is not defined" +else + KAFKA_BROKERS=${KAFKA_BROKERS:-3} + + tries=10 + timeout=1 + while true; do + KAFKA_CHECK=$(kafka-broker-api-versions --bootstrap-server "$CONNECT_BOOTSTRAP_SERVERS" --command-config "${COMMAND_CONFIG_FILE_PATH}" | grep "(id: " | wc -l) + + if [ "$KAFKA_CHECK" -ge "$KAFKA_BROKERS" ]; then + echo "Kafka brokers available." + break + fi + + tries=$((tries - 1)) + if [ ${tries} -eq 0 ]; then + echo "FAILED: KAFKA BROKERs NOT READY." + exit 5 + fi + echo "Expected $KAFKA_BROKERS brokers but found only $KAFKA_CHECK. Waiting $timeout second before retrying ..." + sleep ${timeout} + if [ ${timeout} -lt ${max_timeout} ]; then + timeout=$((timeout * 2)) + fi + done + + echo "Kafka is available." +fi + +if [ -z "$CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL" ]; then + echo "CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL is not defined" +else + tries=10 + timeout=1 + while true; do + if wget --spider -q "${CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL}/subjects" 2>/dev/null; then + echo "Schema registry available." + break + fi + tries=$((tries - 1)) + if [ $tries -eq 0 ]; then + echo "FAILED TO REACH SCHEMA REGISTRY." + exit 6 + fi + echo "Failed to reach schema registry. Retrying in ${timeout} seconds." + sleep ${timeout} + if [ ${timeout} -lt ${max_timeout} ]; then + timeout=$((timeout * 2)) + fi + done + + echo "Schema registry is available." +fi + +if [ $IS_TEMP = 1 ]; then + /bin/rm -f "$COMMAND_CONFIG_FILE_PATH" +fi diff --git a/kafka-connect-upload-source/src/main/docker/kafka-wait b/kafka-connect-upload-source/src/main/docker/kafka-wait deleted file mode 100755 index aa832af2..00000000 --- a/kafka-connect-upload-source/src/main/docker/kafka-wait +++ /dev/null @@ -1,65 +0,0 @@ -#!/bin/bash - -# Check if variables exist -if [ -z "$CONNECT_ZOOKEEPER_CONNECT" ]; then - echo "CONNECT_ZOOKEEPER_CONNECT is not defined" - exit 2 -fi - -if [ -z "$CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL" ]; then - echo "CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL is not defined" - exit 4 -fi - -KAFKA_BROKERS=${KAFKA_BROKERS:-3} - -max_timeout=32 - -tries=10 -timeout=1 -while true; do - ZOOKEEPER_CHECK=$(zookeeper-shell ${CONNECT_ZOOKEEPER_CONNECT} <<< "ls /brokers/ids" | tail -1) - echo "Zookeeper response: ${ZOOKEEPER_CHECK}" - # ZOOKEEPER_CHECK="${ZOOKEEPER_CHECK##*$'\n'}" - ZOOKEEPER_CHECK="$(echo -e "${ZOOKEEPER_CHECK}" | tr -d '[:space:]' | tr -d '[' | tr -d ']')" - - IFS=',' read -r -a array <<< ${ZOOKEEPER_CHECK} - LENGTH=${#array[@]} - if [ "$LENGTH" -ge "$KAFKA_BROKERS" ]; then - echo "Kafka brokers available." - break - fi - - tries=$((tries - 1)) - if [ ${tries} -eq 0 ]; then - echo "FAILED: KAFKA BROKERs NOT READY." - exit 5 - fi - echo "Expected $KAFKA_BROKERS brokers but found only $LENGTH. Waiting $timeout second before retrying ..." - sleep ${timeout} - if [ ${timeout} -lt ${max_timeout} ]; then - timeout=$((timeout * 2)) - fi -done - -tries=10 -timeout=1 -while true; do - if wget --spider -q "${CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL}/subjects" 2>/dev/null; then - echo "Schema registry available." - break - fi - tries=$((tries - 1)) - if [ $tries -eq 0 ]; then - echo "FAILED TO REACH SCHEMA REGISTRY." - exit 6 - fi - echo "Failed to reach schema registry. Retrying in ${timeout} seconds." - sleep ${timeout} - if [ ${timeout} -lt ${max_timeout} ]; then - timeout=$((timeout * 2)) - fi -done - - -echo "Kafka is available. Ready to go!" diff --git a/kafka-connect-upload-source/src/main/docker/launch b/kafka-connect-upload-source/src/main/docker/launch index 9449a466..7ae6b678 100755 --- a/kafka-connect-upload-source/src/main/docker/launch +++ b/kafka-connect-upload-source/src/main/docker/launch @@ -34,19 +34,18 @@ if [ "$KAFKA_JMX_PORT" ]; then export KAFKA_JMX_OPTS="$KAFKA_JMX_OPTS -Djava.rmi.server.hostname=$KAFKA_JMX_HOSTNAME -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.rmi.port=$JMX_PORT -Dcom.sun.management.jmxremote.port=$JMX_PORT" fi -# Busy waiting loop that waits untill all topic are available -echo "===> Wait for infrastructure ..." -kafka-wait -radar_check=$? -if [ "$radar_check" -ne 0 ]; then - exit ${radar_check} -fi - echo "===> Launching ${COMPONENT} ..." # Add our jar to the classpath so that the custom classes can be loaded first. # And this also makes sure that the CLASSPATH does not start with ":/etc/..." # other jars are loaded via the plugin path -export CLASSPATH="/etc/${COMPONENT}/kafka-connect-upload-source/*" +if [ -z "$CLASSPATH" ]; then + export CLASSPATH="/etc/kafka-connect/jars/*" +fi -# execute connector in standalone mode -exec connect-standalone /etc/"${COMPONENT}"/"${COMPONENT}".properties "/etc/${COMPONENT}"/"${CONNECTOR_PROPERTY_FILE_PREFIX}".properties +if [ -z "$CONNECTOR_PROPERTY_FILE_PREFIX" ]; then + # execute connector in distributed mode + exec connect-distributed /etc/"${COMPONENT}"/"${COMPONENT}".properties +else + # execute connector in standalone mode + exec connect-standalone /etc/"${COMPONENT}"/"${COMPONENT}".properties /etc/"${COMPONENT}"/"${CONNECTOR_PROPERTY_FILE_PREFIX}"*.properties +fi diff --git a/kafka-connect-upload-source/src/main/docker/log4j.properties.template b/kafka-connect-upload-source/src/main/docker/log4j.properties.template new file mode 100644 index 00000000..78c4d27b --- /dev/null +++ b/kafka-connect-upload-source/src/main/docker/log4j.properties.template @@ -0,0 +1,32 @@ +# This template file was taken from the Confluent Platform distribution and modified to add Sentry support in Docker images. +# See: https://docs.confluent.io/platform/current/installation/docker/development.html#log-to-external-volumes + +log4j.rootLogger={{ env["CONNECT_LOG4J_ROOT_LOGLEVEL"] | default('INFO') }}, stdout{% if env['SENTRY_DSN'] %}, sentryAppender{% endif %} + + +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern ={{ env["CONNECT_LOG4J_APPENDER_STDOUT_LAYOUT_CONVERSIONPATTERN"] | default('[%d] %p %m (%c)%n') }} + +# Appender for Sentry monitoring +{% if env['SENTRY_DSN'] %} +log4j.appender.sentryAppender=io.sentry.log4j.SentryAppender +log4j.appender.sentryAppender.threshold={{ env['SENTRY_LOG_LEVEL'] | default('ERROR') }} +{% endif %} + +{% set default_loggers = { +'org.reflections': 'ERROR', +'org.apache.zookeeper': 'ERROR', +'org.I0Itec.zkclient': 'ERROR' +} -%} + +{% if env['CONNECT_LOG4J_LOGGERS'] %} +# loggers from CONNECT_LOG4J_LOGGERS env variable +{% set loggers = parse_log4j_loggers(env['CONNECT_LOG4J_LOGGERS']) %} +{% else %} +# default log levels +{% set loggers = default_loggers %} +{% endif %} +{% for logger,loglevel in loggers.items() %} +log4j.logger.{{logger}}={{loglevel}} +{% endfor %} diff --git a/radar-upload-backend/build.gradle.kts b/radar-upload-backend/build.gradle.kts index 9f9cdbbe..1c263a9a 100644 --- a/radar-upload-backend/build.gradle.kts +++ b/radar-upload-backend/build.gradle.kts @@ -1,10 +1,9 @@ plugins { - java application - kotlin("jvm") - id("org.jetbrains.kotlin.plugin.noarg") - id("org.jetbrains.kotlin.plugin.jpa") - id("org.jetbrains.kotlin.plugin.allopen") + id("org.jetbrains.kotlin.plugin.noarg") version Versions.kotlin + id("org.jetbrains.kotlin.plugin.jpa") version Versions.kotlin + id("org.jetbrains.kotlin.plugin.allopen") version Versions.kotlin + id("org.radarbase.radar-kotlin") } application { @@ -12,7 +11,6 @@ application { } dependencies { - api(kotlin("stdlib-jdk8")) implementation(kotlin("reflect")) implementation("org.radarbase:radar-jersey:${Versions.radarJersey}") { @@ -52,3 +50,10 @@ noArg { annotation("jakarta.persistence.MappedSuperclass") annotation("jakarta.persistence.Embeddable") } + +radarKotlin { + // TODO remove after using new release of radar-kotlin plugin + javaVersion.set(Versions.java) + sentryEnabled.set(true) + log4j2Version.set(Versions.log4j2) +} diff --git a/radar-upload-backend/src/main/resources/log4j2.xml b/radar-upload-backend/src/main/resources/log4j2.xml new file mode 100644 index 00000000..71d9c201 --- /dev/null +++ b/radar-upload-backend/src/main/resources/log4j2.xml @@ -0,0 +1,39 @@ + + + + + + + + + + + + + + + + + + + +