From 34fbf9b19f65124c94a7aae4377cd0c36d97cc53 Mon Sep 17 00:00:00 2001 From: Stavros Kontopoulos Date: Thu, 11 May 2017 01:32:27 +0300 Subject: [PATCH 1/2] add ha configuration --- container/appmaster/runit/service/flink/run | 24 +++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/container/appmaster/runit/service/flink/run b/container/appmaster/runit/service/flink/run index 5dde747..0c34f9b 100755 --- a/container/appmaster/runit/service/flink/run +++ b/container/appmaster/runit/service/flink/run @@ -73,6 +73,29 @@ function add_kerberos_configurations() { fi } +function add_ha_configurations() { + if [[ "${FLINK_HA_ENABLED}" == true ]]; then + export FLINK_JAVA_OPTS="$FLINK_JAVA_OPTS -Dhigh-availability=ZOOKEEPER" + add_if_non_empty high-availability.cluster-id "${FLINK_HIGH_AVAILABILITY_CLUSTER_ID}" + add_if_non_empty high-availability.storageDir "${FLINK_HIGH_AVAILABILITY_STORAGE_DIR}" + add_if_non_empty high-availability.zookeeper.quorum "${FLINK_HIGH_AVAILABILITY_ZOOKEEPER_QUORUM}" + add_if_non_empty high-availability.jobmanager.port "${FLINK_HIGH_AVAILABILITY_JOBMANAGER_PORT}" + add_if_non_empty high-availability.job.delay "${FLINK_HIGH_AVAILABILITY_JOB_DELAY}" + add_if_non_empty high-availability.zookeeper.path.root "${FLINK_HIGH_AVAILABILITY_ZOOKEEPER_PATH_ROOT}" + # used for the client at ha services initiated by the mesos app master runner + add_if_non_empty ghigh-availability.zookeeper.client.session-timeout \ + "${FLINK_HIGH_AVAILABILITY_ZOOKEEPER_CLIENT_SESSION_TIMEOUT}" + add_if_non_empty high-availability.zookeeper.client.connection-timeout \ + "${FLINK_HIGH_AVAILABILITY_ZOOKEEPER_CLIENT_CONNECTION_TIMEOUT}" + add_if_non_empty high-availability.zookeeper.client.retry-wait \ + "${FLINK_HIGH_AVAILABILITY_ZOOKEEPER_CLIENT_RETRY_WAIT}" + add_if_non_empty high-availability.zookeeper.client.max-retry-attempts \ + "${FLINK_HIGH_AVAILABILITY_ZOOKEEPER_CLIENT_MAX_RETRY_ATTEMPTS}" + add_if_non_empty high-availability.zookeeper.path.running-registry \ + "${FLINK_HIGH_AVAILABILITY_ZOOKEEPER_PATH_RUNNING_REGISTRY}" + fi +} + function update_log_level() { if [[ "${FLINK_LOG_LEVEL}" != "" ]]; then sed -ie 's/log4j.rootLogger=INFO, file/log4j.rootLogger='"${FLINK_LOG_LEVEL}"', file/g' ${FLINK_HOME}/conf/log4j.properties @@ -80,6 +103,7 @@ function update_log_level() { } add_flink_configurations +add_ha_configurations add_mesos_configurations add_ssl_configurations add_kerberos_configurations From ed7bb4ff029f704279e6e7a1089f8f676d1ed7ea Mon Sep 17 00:00:00 2001 From: Stavros Kontopoulos Date: Tue, 1 Aug 2017 16:44:41 +0300 Subject: [PATCH 2/2] fix typo --- container/appmaster/runit/service/flink/run | 43 ++++++++++++++------- 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/container/appmaster/runit/service/flink/run b/container/appmaster/runit/service/flink/run index 947007c..268c33c 100755 --- a/container/appmaster/runit/service/flink/run +++ b/container/appmaster/runit/service/flink/run @@ -33,8 +33,14 @@ function add_if_non_empty() { fi } +function add_if_non_empty_to_config() { + if [ -n "$2" ]; then + echo "$1: $2" >> ${FLINK_HOME}/conf/flink-conf.yaml + fi +} + function add_mesos_configurations() { - add_if_non_empty jobmanager.rpc.address $(hostname -f) + add_if_non_empty jobmanager.rpc.address $(hostname -f) add_if_non_empty mesos.resourcemanager.framework.role "${FLINK_DISPATCHER_MESOS_ROLE}" add_if_non_empty mesos.resourcemanager.framework.principal "${FLINK_DISPATCHER_MESOS_PRINCIPAL}" add_if_non_empty mesos.resourcemanager.framework.secret "${FLINK_DISPATCHER_MESOS_SECRET}" @@ -76,24 +82,33 @@ function add_kerberos_configurations() { function add_ha_configurations() { if [[ "${FLINK_HA_ENABLED}" == true ]]; then - export FLINK_JAVA_OPTS="$FLINK_JAVA_OPTS -Dhigh-availability=ZOOKEEPER" - add_if_non_empty high-availability.cluster-id "${FLINK_HIGH_AVAILABILITY_CLUSTER_ID}" - add_if_non_empty high-availability.storageDir "${FLINK_HIGH_AVAILABILITY_STORAGE_DIR}" - add_if_non_empty high-availability.zookeeper.quorum "${FLINK_HIGH_AVAILABILITY_ZOOKEEPER_QUORUM}" - add_if_non_empty high-availability.jobmanager.port "${FLINK_HIGH_AVAILABILITY_JOBMANAGER_PORT}" - add_if_non_empty high-availability.job.delay "${FLINK_HIGH_AVAILABILITY_JOB_DELAY}" - add_if_non_empty high-availability.zookeeper.path.root "${FLINK_HIGH_AVAILABILITY_ZOOKEEPER_PATH_ROOT}" + add_if_non_empty_to_config high-availability ZOOKEEPER + add_if_non_empty_to_config high-availability.cluster-id "${FLINK_HIGH_AVAILABILITY_CLUSTER_ID}" + add_if_non_empty_to_config high-availability.storageDir "${FLINK_HIGH_AVAILABILITY_STORAGE_DIR}" + add_if_non_empty_to_config high-availability.zookeeper.quorum "${FLINK_HIGH_AVAILABILITY_ZOOKEEPER_QUORUM}" + add_if_non_empty_to_config high-availability.jobmanager.port "${FLINK_HIGH_AVAILABILITY_JOBMANAGER_PORT}" + add_if_non_empty_to_config high-availability.job.delay "${FLINK_HIGH_AVAILABILITY_JOB_DELAY}" + add_if_non_empty_to_config high-availability.zookeeper.path.root "${FLINK_HIGH_AVAILABILITY_ZOOKEEPER_PATH_ROOT}" + add_if_non_empty_to_config high-availability.zookeeper.path.running-registry \ + "${FLINK_HIGH_AVAILABILITY_ZOOKEEPER_PATH_RUNNING_REGISTRY}" + add_if_non_empty_to_config high-availability.zookeeper.path.latch "${FLINK_HIGH_AVAILABILITY_ZOOKEEPER_PATH_LEADER_LATCH}" + add_if_non_empty_to_config high-availability.zookeeper.path.jobgraphs "${FLINK_HIGH_AVAILABILITY_ZOOKEEPER_PATH_JOBGRAPHS}" + add_if_non_empty_to_config high-availability.zookeeper.path.leader "${FLINK_HIGH_AVAILABILITY_ZOOKEEPER_PATH_LEADER}" + add_if_non_empty_to_config high-availability.zookeeper.path.checkpoints "${FLINK_HIGH_AVAILABILITY_ZOOKEEPER_PATH_CHECKPOINTS}" + add_if_non_empty_to_config high-availability.zookeeper.path.checkpoint-counter "${FLINK_HIGH_AVAILABILITY_ZOOKEEPER_PATH_CHECKPOINT_COUNTER}" + add_if_non_empty_to_config high-availability.zookeeper.path.mesos-workers "${FLINK_HIGH_AVAILABILITY_ZOOKEEPER_PATH_MESOS_WORKERS}" + # used for the client at ha services initiated by the mesos app master runner - add_if_non_empty ghigh-availability.zookeeper.client.session-timeout \ + add_if_non_empty_to_config high-availability.zookeeper.client.session-timeout \ "${FLINK_HIGH_AVAILABILITY_ZOOKEEPER_CLIENT_SESSION_TIMEOUT}" - add_if_non_empty high-availability.zookeeper.client.connection-timeout \ + add_if_non_empty_to_config high-availability.zookeeper.client.connection-timeout \ "${FLINK_HIGH_AVAILABILITY_ZOOKEEPER_CLIENT_CONNECTION_TIMEOUT}" - add_if_non_empty high-availability.zookeeper.client.retry-wait \ + add_if_non_empty_to_config high-availability.zookeeper.client.retry-wait \ "${FLINK_HIGH_AVAILABILITY_ZOOKEEPER_CLIENT_RETRY_WAIT}" - add_if_non_empty high-availability.zookeeper.client.max-retry-attempts \ + add_if_non_empty_to_config high-availability.zookeeper.client.max-retry-attempts \ "${FLINK_HIGH_AVAILABILITY_ZOOKEEPER_CLIENT_MAX_RETRY_ATTEMPTS}" - add_if_non_empty high-availability.zookeeper.path.running-registry \ - "${FLINK_HIGH_AVAILABILITY_ZOOKEEPER_PATH_RUNNING_REGISTRY}" + add_if_non_empty_to_config high-availability.zookeeper.client.acl \ + "${FLINK_HIGH_AVAILABILITY_ZOOKEEPER_CLIENT_ACL}" fi }