From 5b66e703e24b8387bbfe5aa66c081497574c9308 Mon Sep 17 00:00:00 2001 From: KARTHICK Duraisamy Soundararaj Date: Fri, 21 Apr 2017 15:43:11 -0700 Subject: [PATCH 1/4] Changes to workaround timeouts while connecting to consul container due to network limiations imposed by xhyve based native docker on mac. Please see http://tinyurl.com/z4p46k6 --- ambari-functions | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/ambari-functions b/ambari-functions index 901a1c0..8d9d1c9 100644 --- a/ambari-functions +++ b/ambari-functions @@ -43,13 +43,17 @@ get-consul-ip() { get-host-ip $CONSUL } +get-consul-base-url() { + echo "http://127.0.0.1:8500" +} + get-host-ip() { HOST=$1 docker inspect --format="{{.NetworkSettings.IPAddress}}" ${HOST} } amb-members() { - curl http://$(get-consul-ip):8500/v1/catalog/nodes | sed -e 's/,{"Node":"ambari-8080.*}//g' -e 's/,{"Node":"consul.*}//g' + curl $(get-consul-base-url)/v1/catalog/nodes | sed -e 's/,{"Node":"ambari-8080.*}//g' -e 's/,{"Node":"consul.*}//g' } amb-settings() { @@ -127,10 +131,10 @@ amb-start-first() { if [[ "$EXPOSE_DNS" == "true" ]]; then dns_port_command="-p 53:$DNS_PORT/udp" fi - run-command docker run -d $dns_port_command --name $CONSUL -h $CONSUL.service.consul $CONSUL_IMAGE -server -bootstrap + run-command docker run -p 8500:8500 -d $dns_port_command --name $CONSUL -h $CONSUL.service.consul $CONSUL_IMAGE -server -bootstrap sleep 5 - run-command docker run -d $DOCKER_OPTS --privileged --name $AMBARI_SERVER_NAME -h $AMBARI_SERVER_NAME.service.consul $AMBARI_SERVER_IMAGE \ + run-command docker run -p 8080:8080 -d $DOCKER_OPTS --privileged --name $AMBARI_SERVER_NAME -h $AMBARI_SERVER_NAME.service.consul $AMBARI_SERVER_IMAGE \ systemd.setenv=NAMESERVER_ADDR=$(get-consul-ip) get-ambari-server-ip @@ -187,5 +191,5 @@ _consul-register-service() { \"Service\": { \"Service\": \"$1\" } - }" http://$(get-consul-ip):8500/v1/catalog/register + }" $(get-consul-base-url)/v1/catalog/register } From d6e8d466a2dc7c692d85cac8486b6a13b7e8ba21 Mon Sep 17 00:00:00 2001 From: KARTHICK Duraisamy Soundararaj Date: Tue, 2 May 2017 12:09:44 -0700 Subject: [PATCH 2/4] Changes to decouple nameserver (re)configuration --- ambari-agent/Dockerfile | 4 - ambari-agent/docker_shared/etc/resolv.conf | 2 + .../{init => docker_shared}/init-agent.sh | 26 +---- ambari-agent/init/ambari-agent.service | 2 +- ambari-functions | 100 ++++++++++++++++-- ambari-server/Dockerfile | 3 - ambari-server/docker_shared/etc/resolv.conf | 2 + .../{init => docker_shared}/init-server.sh | 26 +---- ambari-server/init/ambari-server.service | 2 +- 9 files changed, 104 insertions(+), 63 deletions(-) create mode 100644 ambari-agent/docker_shared/etc/resolv.conf rename ambari-agent/{init => docker_shared}/init-agent.sh (56%) create mode 100644 ambari-server/docker_shared/etc/resolv.conf rename ambari-server/{init => docker_shared}/init-server.sh (82%) diff --git a/ambari-agent/Dockerfile b/ambari-agent/Dockerfile index 92ceb0c..ba2a7de 100644 --- a/ambari-agent/Dockerfile +++ b/ambari-agent/Dockerfile @@ -5,7 +5,6 @@ MAINTAINER Hortonworks ADD ambari.repo /etc/yum.repos.d/ RUN yum install -y ambari-agent && yum clean all -#RUN find /etc/rc.d/rc* -name "*ambari-agent" | xargs rm -v # add a custom folder to the hadoop classpath RUN mkdir -p /usr/lib/hadoop/lib @@ -23,9 +22,6 @@ RUN sed -i "s/\"ifconfig\"/\"ifconfig eth0\"/" /usr/lib/python2.6/site-packages/ ADD dash-azure-storage-2.2.0.jar /usr/lib/hadoop/lib/ ADD gcs-connector-latest-hadoop2.jar /usr/lib/hadoop/lib/ -ADD init/init-agent.sh /opt/ambari-agent/init-agent.sh -RUN chmod u+x /opt/ambari-agent/init-agent.sh - ADD init/ambari-agent.service /etc/systemd/system/ambari-agent.service RUN systemctl enable ambari-agent diff --git a/ambari-agent/docker_shared/etc/resolv.conf b/ambari-agent/docker_shared/etc/resolv.conf new file mode 100644 index 0000000..a8e7a07 --- /dev/null +++ b/ambari-agent/docker_shared/etc/resolv.conf @@ -0,0 +1,2 @@ +nameserver 172.17.0.2 +search service.consul node.dc1.consul diff --git a/ambari-agent/init/init-agent.sh b/ambari-agent/docker_shared/init-agent.sh similarity index 56% rename from ambari-agent/init/init-agent.sh rename to ambari-agent/docker_shared/init-agent.sh index 055de17..78a65f8 100755 --- a/ambari-agent/init/init-agent.sh +++ b/ambari-agent/docker_shared/init-agent.sh @@ -10,29 +10,6 @@ debug() { [[ "DEBUG" ]] && echo "[DEBUG] $@" 1>&2 } -get_nameserver_addr() { - if [[ "$NAMESERVER_ADDR" ]]; then - echo $NAMESERVER_ADDR - else - if ip addr show docker0 &> /dev/null; then - ip addr show docker0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1 - else - ip ro | grep default | cut -d" " -f 3 - fi - fi -} - -# --dns isn't available for: docker run --net=host -# sed -i /etc/resolf.conf fails: -# sed: cannot rename /etc/sedU9oCRy: Device or resource busy -# here comes the tempfile workaround ... -local_nameserver() { - cat>/etc/resolv.conf<$WORK_DIR/ambari_server_shared/etc/resolv.conf<$WORK_DIR/ambari_agent_shared/etc/resolv.conf<&2 } -get_nameserver_addr() { - if [[ "$NAMESERVER_ADDR" ]]; then - echo $NAMESERVER_ADDR - else - if ip addr show docker0 &> /dev/null; then - ip addr show docker0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1 - else - ip ro | grep default | cut -d" " -f 3 - fi - fi -} - -# --dns isn't available for: docker run --net=host -# sed -i /etc/resolf.conf fails: -# sed: cannot rename /etc/sedU9oCRy: Device or resource busy -# here comes the tempfile workaround ... -local_nameserver() { - cat>/etc/resolv.conf< Date: Tue, 2 May 2017 12:17:12 -0700 Subject: [PATCH 3/4] Documentation & Script to automate image builds --- build-docker-images.sh | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 build-docker-images.sh diff --git a/build-docker-images.sh b/build-docker-images.sh new file mode 100644 index 0000000..25a3d98 --- /dev/null +++ b/build-docker-images.sh @@ -0,0 +1,2 @@ +cd ambari-agent/ && docker build -t hortonworks/ambari-agent:latest . && cd .. +cd ambari-server/ && docker build -t hortonworks/ambari-server:latest . && cd .. From 651f9a658ba91713962e5dc3ee093cfd9ba24869 Mon Sep 17 00:00:00 2001 From: KARTHICK Duraisamy Soundararaj Date: Thu, 11 May 2017 12:41:18 -0700 Subject: [PATCH 4/4] Kerberos for hadoop --- ambari-functions | 32 +++++++-- docker-kerberos/Dockerfile | 8 +++ docker-kerberos/README.md | 45 ++++++++++++ .../docker_shared/init-kerberos.sh | 68 +++++++++++++++++++ docker-kerberos/docker_shared/krb5.conf | 22 ++++++ docker_shared/etc/krb5.conf | 22 ++++++ docker_shared/etc/resolv.conf | 2 + 7 files changed, 192 insertions(+), 7 deletions(-) create mode 100644 docker-kerberos/Dockerfile create mode 100644 docker-kerberos/README.md create mode 100755 docker-kerberos/docker_shared/init-kerberos.sh create mode 100644 docker-kerberos/docker_shared/krb5.conf create mode 100644 docker_shared/etc/krb5.conf create mode 100644 docker_shared/etc/resolv.conf diff --git a/ambari-functions b/ambari-functions index e6096e9..270f6bb 100644 --- a/ambari-functions +++ b/ambari-functions @@ -9,10 +9,13 @@ USAGE : ${WORK_DIR:="/tmp/work_dir/"} : ${AMBARI_SERVER_SHARED:="${WORK_DIR}/ambari_server_shared"} : ${AMBARI_AGENT_SHARED:="${WORK_DIR}/ambari_agent_shared"} +: ${KERBEROS:="kerberos"} +: ${KERBEROS_SHARED:="${WORK_DIR}/kerberos_shared"} : ${NODE_PREFIX=amb} : ${AMBARI_SERVER_NAME:=${NODE_PREFIX}-server} : ${AMBARI_SERVER_IMAGE:="hortonworks/ambari-server:latest"} : ${AMBARI_AGENT_IMAGE:="hortonworks/ambari-agent:latest"} +: ${KERBEROS_IMAGE:="sequenceiq/kerberos:latest"} : ${DOCKER_OPTS:=""} : ${CONSUL:="${NODE_PREFIX}-consul"} : ${CONSUL_IMAGE:="sequenceiq/consul:v0.5.0-v6"} @@ -27,8 +30,13 @@ USAGE init() { mkdir -p $WORK_DIR mkdir -p $AMBARI_SERVER_SHARED + + rm -rf {ambari-server,ambari-agent,docker-kerberos}/docker_shared/ + cp -r docker_shared/etc {ambari-server,ambari-agent,docker-kerberos}/docker_shared/ + cp -rf ambari-server/docker_shared/ $AMBARI_SERVER_SHARED cp -rf ambari-agent/docker_shared/ $AMBARI_AGENT_SHARED + cp -rf docker-kerberos/docker_shared/ $KERBEROS_SHARED } update-nameserver() { @@ -228,6 +236,15 @@ amb-start-first() { _consul-register-service ambari-8080 $AMBARI_SERVER_IP } +kerberos-start-first() { + echo run-command docker run -v $KERBEROS_SHARED:/docker_shared -v /dev/urandom:/dev/random -h kerberos.service.consul --name $KERBEROS -e BOOTSTRAP=0 --security-opt seccomp=unconfined -dt $DOCKER_OPTS --entrypoint /docker_shared/init-kerberos.sh $KERBEROS_IMAGE + run-command docker run -v $KERBEROS_SHARED:/docker_shared -v /dev/urandom:/dev/random -h kerberos.service.consul --name $KERBEROS -e BOOTSTRAP=0 --security-opt seccomp=unconfined -dt $DOCKER_OPTS --entrypoint /docker_shared/init-kerberos.sh $KERBEROS_IMAGE + sleep 10 + kerberos_ip=$(get-host-ip $KERBEROS) + echo "Kerberos running on" $kerberos_ip + _consul-register-service $KERBEROS $kerberos_ip +} + amb-copy-to-hdfs() { get-ambari-server-ip FILE_PATH=${1:?"usage: "} @@ -269,11 +286,12 @@ amb-start-node() { } _consul-register-service() { - curl -X PUT -d "{ - \"Node\": \"$1\", - \"Address\": \"$2\", - \"Service\": { - \"Service\": \"$1\" - } - }" $(get-consul-base-url)/v1/catalog/register + echo "Dummy Registration" + # curl -X PUT -d "{ + # \"Node\": \"$1\", + # \"Address\": \"$2\", + # \"Service\": { + # \"Service\": \"$1\" + # } + #}" $(get-consul-base-url)/v1/catalog/register } diff --git a/docker-kerberos/Dockerfile b/docker-kerberos/Dockerfile new file mode 100644 index 0000000..6023a93 --- /dev/null +++ b/docker-kerberos/Dockerfile @@ -0,0 +1,8 @@ +FROM centos:6.6 +MAINTAINER SequenceIQ + +# EPEL +RUN rpm -Uvh http://dl.fedoraproject.org/pub/epel/6/x86_64/epel-release-6-8.noarch.rpm + +# kerberos +RUN yum install -y krb5-server krb5-libs krb5-auth-dialog krb5-workstation diff --git a/docker-kerberos/README.md b/docker-kerberos/README.md new file mode 100644 index 0000000..9312f5f --- /dev/null +++ b/docker-kerberos/README.md @@ -0,0 +1,45 @@ +## Docker kerberos +This image +is designed to support the Hadoop clusters launched by Cloudbreak. The default realm is `NODE.DC1.CONSUL` and the default admin principal is `admin/admin`. All the default values can be modified with environment variables. + +### Usage + +The image can be started in bootstrap mode and non-bootstrap mode. Bootstrap mode means +when the container is launched it will create the DB for kerberos along with the admin user and start the KDC. +This use-case is convenient for a quick start. The non-bootstrap mode relies on that a third party will do the necessary steps to create the appropriate principals thus the KDC will start only once they are created. +Cloudbreak does this with a [consul plugn](https://github.com/sequenceiq/consul-plugins-kerberos). + +#### Quick start +``` +docker run -d --net=host -v /etc/krb5.conf:/etc/krb5.conf -v /dev/urandom:/dev/random --name kerberos -e BOOTSTRAP=0 sequenceiq/kerberos +``` +The containers have a pretty bad entropy level so the KDC won't start because of this. We can overcome this by using `/dev/urandom` which is less secure but does not care about entropy. +The `/etc/krb5.conf` is shared with the host so the generated configuration will be present on the host as well. We need to share this configuration with the `ambari-server` container as well or you need to take care of the copying. +Once the container is running you can enable kerberos with `Ambari`. + +Useful environment variables: + +| Environmenr variables | Description | +| --------------------- | ----------------------------- | +| `REALM` | the Kerberos realm | +| `DOMAIN_REALM` | the DNS domain for the realm | +| `KERB_MASTER_KEY` | master key for the KDC | +| `KERB_ADMIN_USER` | administrator account name | +| `KERB_ADMIN_PASS` | administrator's password | +| `SEARCH_DOMAINS` | domain suffix search list | + +### Test +Once kerberos is enabled you need a `ticket` to execute any job on the cluster. Here's an example to get a ticket: +``` +kinit -V -kt /etc/security/keytabs/smokeuser.headless.keytab ambari-qa-sparktest-rec@NODE.DC1.CONSUL +``` +Example job: +```java +export HADOOP_LIBS=/usr/hdp/current/hadoop-mapreduce-client +export JAR_EXAMPLES=$HADOOP_LIBS/hadoop-mapreduce-examples.jar +export JAR_JOBCLIENT=$HADOOP_LIBS/hadoop-mapreduce-client-jobclient.jar + +hadoop jar $JAR_EXAMPLES teragen 10000000 /user/ambari-qa/terasort-input + +hadoop jar $JAR_JOBCLIENT mrbench -baseDir /user/ambari-qa/smallJobsBenchmark -numRuns 5 -maps 10 -reduces 5 -inputLines 10 -inputType ascending +``` diff --git a/docker-kerberos/docker_shared/init-kerberos.sh b/docker-kerberos/docker_shared/init-kerberos.sh new file mode 100755 index 0000000..34236cf --- /dev/null +++ b/docker-kerberos/docker_shared/init-kerberos.sh @@ -0,0 +1,68 @@ +#!/bin/bash + +: ${KERB_MASTER_KEY:=masterkey} +: ${REALM:=SERVICE.CONSUL} +: ${KERB_ADMIN_USER:=admin} +: ${KERB_ADMIN_PASS:=admin} + +create_config() { + cp /docker_shared/krb5.conf /etc/krb5.conf +} + +create_db() { + /usr/sbin/kdb5_util -P $KERB_MASTER_KEY -r $REALM create -s +} + +create_admin_user() { + kadmin.local -q "addprinc -pw $KERB_ADMIN_PASS $KERB_ADMIN_USER/admin" + echo "*/admin@$REALM *" > /var/kerberos/krb5kdc/kadm5.acl +} + +fix_hostname() { + sed -i "/^hosts:/ s/ *files dns/ dns files/" /etc/nsswitch.conf +} + +create_db() { + /usr/sbin/kdb5_util -P $KERB_MASTER_KEY -r $REALM create -s +} + +start_kdc() { + mkdir -p /var/log/kerberos + + chkconfig krb5kdc on + chkconfig kadmin on + + /sbin/service krb5kdc start #/etc/rc.d/init.d/krb5kdc start + /sbin/service kadmin start #/etc/rc.d/init.d/kadmin start +} + +restart_kdc() { + /etc/rc.d/init.d/krb5kdc restart + /etc/rc.d/init.d/kadmin restart +} + +create_admin_user() { + kadmin.local -q "addprinc -pw $KERB_ADMIN_PASS $KERB_ADMIN_USER/admin" + echo "*/admin@$REALM *" > /var/kerberos/krb5kdc/kadm5.acl +} + +main() { + ln -s /docker_shared/etc/resolv.conf /tmp/resolv.conf + cp /tmp/resolv.conf /etc/resolv.conf + + mkdir -p /var/log/kerberos + + if [ ! -f /var/kerberos/kerberos_initialized ]; then + create_config + create_db + create_admin_user + start_kdc + + touch /var/kerberos/kerberos_initialized + fi + + tail -f /dev/null + #fix_hostname +} + +[[ "$0" == "$BASH_SOURCE" ]] && main "$@" diff --git a/docker-kerberos/docker_shared/krb5.conf b/docker-kerberos/docker_shared/krb5.conf new file mode 100644 index 0000000..f3bcd7f --- /dev/null +++ b/docker-kerberos/docker_shared/krb5.conf @@ -0,0 +1,22 @@ +[logging] + default = FILE:/var/log/kerberos/krb5libs.log + kdc = FILE:/var/log/kerberos/krb5kdc.log + admin_server = FILE:/var/log/kerberos/kadmind.log + +[libdefaults] + default_realm = SERVICE.CONSUL + dns_lookup_realm = false + dns_lookup_kdc = false + ticket_lifetime = 24h + renew_lifetime = 7d + forwardable = true + +[realms] + SERVICE.CONSUL = { + kdc = kerberos.service.consul + admin_server = kerberos.service.consul + } + +[domain_realm] + .service.consul = SERVICE.CONSUL + service.consul = SERVICE.CONSUL diff --git a/docker_shared/etc/krb5.conf b/docker_shared/etc/krb5.conf new file mode 100644 index 0000000..ef3be51 --- /dev/null +++ b/docker_shared/etc/krb5.conf @@ -0,0 +1,22 @@ +[logging] + default = FILE:/var/log/kerberos/krb5libs.log + kdc = FILE:/var/log/kerberos/krb5kdc.log + admin_server = FILE:/var/log/kerberos/kadmind.log + +[libdefaults] + default_realm = DSE.GROUP.ON + dns_lookup_realm = false + dns_lookup_kdc = false + ticket_lifetime = 24h + renew_lifetime = 7d + forwardable = true + +[realms] + DSE.GROUP.ON = { + kdc = kerberos.service.consul + admin_server = kerberos.service.consul + } + +[domain_realm] + .service.consul = DSE.GROUP.ON + service.consul = DSE.GROUP.ON diff --git a/docker_shared/etc/resolv.conf b/docker_shared/etc/resolv.conf new file mode 100644 index 0000000..a8e7a07 --- /dev/null +++ b/docker_shared/etc/resolv.conf @@ -0,0 +1,2 @@ +nameserver 172.17.0.2 +search service.consul node.dc1.consul