From 642d666bf3ab2c97258c60d116224581c601c650 Mon Sep 17 00:00:00 2001 From: "Susan X. Huynh" Date: Mon, 27 Aug 2018 13:22:28 -0700 Subject: [PATCH 01/10] With this image, I can repro the Hive Sentry CREATETABLE bug and show that the Lightbend PR fixes the bug. --- tools/hive/hadoop-hive/Dockerfile | 10 ++++ .../hadoop-hive/scripts/hive-bootstrap.sh | 19 +++++-- .../templates/hive-site.xml.template | 37 ++++++++++++++ tools/hive/kerberos/Dockerfile | 8 +++ .../kerberos/marathon/hdfs-hive-kerberos.json | 2 +- .../hive/kerberos/scripts/test-hive-sentry.sh | 31 ++++++++++++ .../templates/hive-site-kerberos.xml.template | 6 +-- .../sentry-site.xml.hive-client.template | 44 ++++++++++++++++ .../templates/sentry-site.xml.server.template | 50 +++++++++++++++++++ 9 files changed, 200 insertions(+), 7 deletions(-) create mode 100644 tools/hive/kerberos/scripts/test-hive-sentry.sh create mode 100644 tools/hive/kerberos/templates/sentry-site.xml.hive-client.template create mode 100644 tools/hive/kerberos/templates/sentry-site.xml.server.template diff --git a/tools/hive/hadoop-hive/Dockerfile b/tools/hive/hadoop-hive/Dockerfile index c7a97d53..d3ef1c9b 100644 --- a/tools/hive/hadoop-hive/Dockerfile +++ b/tools/hive/hadoop-hive/Dockerfile @@ -17,6 +17,7 @@ ENV POSTGRESQL_MAIN /var/lib/postgresql/9.5/main/ ENV POSTGRESQL_CONFIG_FILE /var/lib/postgresql/9.5/main/postgresql.conf ENV POSTGRESQL_BIN /usr/lib/postgresql/9.5/bin/postgres ENV PGPASSWORD hive +ENV SENTRY_HOME /usr/local/sentry # install dev tools RUN apt-get update && \ @@ -145,5 +146,14 @@ RUN chmod 700 /etc/hive-bootstrap.sh EXPOSE 10000 10001 10002 10003 9083 50111 5432 +# sentry +RUN curl -L http://archive.cloudera.com/cdh5/cdh/5/sentry-1.5.1-cdh5.11.0.tar.gz | tar -xzC /usr/local && \ + cd /usr/local && \ + ln -s apache-sentry-1.5.1-cdh5.11.0-bin/ sentry +ADD templates/sentry-site.xml.hive-client.template /usr/local/hive/conf/sentry-site.xml.template +ADD templates/sentry-site.xml.server.template /usr/local/sentry/conf/sentry-site.xml.template +ADD scripts/test-hive-sentry.sh /etc/test-hive-sentry.sh +RUN chmod 700 /etc/test-hive-sentry.sh + # run bootstrap script CMD ["/etc/hive-bootstrap.sh", "-d"] diff --git a/tools/hive/hadoop-hive/scripts/hive-bootstrap.sh b/tools/hive/hadoop-hive/scripts/hive-bootstrap.sh index 4d40308f..126d5fe0 100644 --- a/tools/hive/hadoop-hive/scripts/hive-bootstrap.sh +++ b/tools/hive/hadoop-hive/scripts/hive-bootstrap.sh @@ -6,6 +6,12 @@ printenv | cat >> /root/.bashrc # hadoop bootstrap /etc/hadoop-bootstrap.sh -d +# init and start sentry +SENTRY_CONF_FILE=$SENTRY_HOME/conf/sentry-site.xml +sed s/{{HOSTNAME}}/$HOSTNAME/ $SENTRY_HOME/conf/sentry-site.xml.template > $SENTRY_HOME/conf/sentry-site.xml +$SENTRY_HOME/bin/sentry --command schema-tool --conffile $SENTRY_CONF_FILE --dbType derby --initSchema +$SENTRY_HOME/bin/sentry --command service --conffile $SENTRY_CONF_FILE & + # restart postgresql /etc/init.d/postgresql restart @@ -19,16 +25,23 @@ do echo "waiting for hdfs to be ready"; sleep 10; done +# create hive user +useradd hive + # create hdfs directories -$HADOOP_PREFIX/bin/hdfs dfs -mkdir -p /user/root +hdfs dfs -mkdir -p /user/root hdfs dfs -chown -R hdfs:supergroup /user -$HADOOP_PREFIX/bin/hdfs dfs -mkdir -p /apps/hive/warehouse +hdfs dfs -mkdir -p /apps/hive/warehouse hdfs dfs -chown -R hive:supergroup /apps/hive hdfs dfs -chmod 777 /apps/hive/warehouse +hdfs dfs -mkdir -p /tmp/hive +hdfs dfs -chmod 777 /tmp/hive + # altering the hive-site configuration -sed s/{{HOSTNAME}}/$HOSTNAME/ /usr/local/hive/conf/hive-site.xml.template > /usr/local/hive/conf/hive-site.xml +sed s/{{HOSTNAME}}/$HOSTNAME/ $HIVE_CONF/hive-site.xml.template > $HIVE_CONF/hive-site.xml +sed s/{{HOSTNAME}}/$HOSTNAME/ $HIVE_CONF/sentry-site.xml.template > $HIVE_CONF/sentry-site.xml sed s/{{HOSTNAME}}/$HOSTNAME/ /opt/files/hive-site.xml.template > /opt/files/hive-site.xml # start hive metastore server diff --git a/tools/hive/hadoop-hive/templates/hive-site.xml.template b/tools/hive/hadoop-hive/templates/hive-site.xml.template index 8e61b910..6127d942 100755 --- a/tools/hive/hadoop-hive/templates/hive-site.xml.template +++ b/tools/hive/hadoop-hive/templates/hive-site.xml.template @@ -20,4 +20,41 @@ hive.metastore.warehouse.dir /apps/hive/warehouse + + + + hive.server2.enable.doAs + false + + + + hive.security.authorization.task.factory + org.apache.sentry.binding.hive.SentryHiveAuthorizationTaskFactoryImpl + + + + hive.server2.session.hook + org.apache.sentry.binding.hive.HiveAuthzBindingSessionHook + + + + hive.sentry.conf.url + file:///usr/local/hive/conf/sentry-site.xml + + + + + hive.metastore.filter.hook + org.apache.sentry.binding.metastore.SentryMetaStoreFilterHook + + + + hive.metastore.pre.event.listeners + org.apache.sentry.binding.metastore.MetastoreAuthzBinding + + + + hive.metastore.event.listeners + org.apache.sentry.binding.metastore.SentryMetastorePostEventListener + diff --git a/tools/hive/kerberos/Dockerfile b/tools/hive/kerberos/Dockerfile index 19559f65..d2f84c34 100644 --- a/tools/hive/kerberos/Dockerfile +++ b/tools/hive/kerberos/Dockerfile @@ -9,6 +9,14 @@ ADD templates/yarn-site.xml.template $HADOOP_PREFIX/etc/hadoop/yarn-site.xml.tem ADD templates/hive-site.xml.template /opt/files/ ADD templates/hive-site.xml.template $HIVE_CONF/hive-site.xml.template +# kerberized sentry config files +ADD templates/sentry-site.xml.hive-client.template /usr/local/hive/conf/sentry-site.xml.template +ADD templates/sentry-site.xml.server.template /usr/local/sentry/conf/sentry-site.xml.template + +# hive / sentry test script +ADD scripts/test-hive-sentry.sh /etc/test-hive-sentry.sh +RUN chmod 700 /etc/test-hive-sentry.sh + # krb5.conf ADD conf/krb5.conf /etc/ diff --git a/tools/hive/kerberos/marathon/hdfs-hive-kerberos.json b/tools/hive/kerberos/marathon/hdfs-hive-kerberos.json index 8cfa91ac..1360bb0b 100644 --- a/tools/hive/kerberos/marathon/hdfs-hive-kerberos.json +++ b/tools/hive/kerberos/marathon/hdfs-hive-kerberos.json @@ -33,7 +33,7 @@ [ "hostname", "IS", - "10.0.0.114" + "10.0.1.160" ] ] } diff --git a/tools/hive/kerberos/scripts/test-hive-sentry.sh b/tools/hive/kerberos/scripts/test-hive-sentry.sh new file mode 100644 index 00000000..f5e0dd89 --- /dev/null +++ b/tools/hive/kerberos/scripts/test-hive-sentry.sh @@ -0,0 +1,31 @@ +#!/bin/bash +set -x + +# Create a user "alice" since Sentry authorization relies on the Linux user and group information +useradd alice + +# Grant permissions to user “alice” +echo "Grant permissions to user alice ..." +kdestroy +kinit -k -t /usr/local/hadoop/etc/hadoop/hdfs.keytab hive/${HOSTNAME}@LOCAL +cat <grant_alice.sql +CREATE ROLE test_role; +GRANT ROLE test_role to GROUP alice; +GRANT ROLE test_role to GROUP root; +GRANT ALL on DATABASE default to ROLE test_role WITH GRANT OPTION; +EOF +beeline -u "jdbc:hive2://localhost:10000/default;principal=hive/${HOSTNAME}@LOCAL" -f grant_alice.sql + +# Test Hive / Sentry +#echo "Create a table in Hive with Sentry as alice ..." +#kdestroy +#kinit -k -t /usr/local/hadoop/etc/hadoop/hdfs.keytab alice/${HOSTNAME}@LOCAL +#cat <create_table.sql +#CREATE TABLE test1 (col1 INT); +#SHOW TABLES; +#EOF +#beeline -u "jdbc:hive2://localhost:10000/default;principal=alice/${HOSTNAME}@LOCAL" -f create_table.sql + +# Log back in as hdfs +kdestroy +kinit -k -t /usr/local/hadoop/etc/hadoop/hdfs.keytab hdfs@LOCAL diff --git a/tools/hive/kerberos/templates/hive-site-kerberos.xml.template b/tools/hive/kerberos/templates/hive-site-kerberos.xml.template index 5dad8f1f..a500c6c2 100755 --- a/tools/hive/kerberos/templates/hive-site-kerberos.xml.template +++ b/tools/hive/kerberos/templates/hive-site-kerberos.xml.template @@ -38,11 +38,11 @@ hive.users.in.admin.role - hdfs,hive + hive - + diff --git a/tools/hive/kerberos/templates/sentry-site.xml.hive-client.template b/tools/hive/kerberos/templates/sentry-site.xml.hive-client.template new file mode 100644 index 00000000..393d0118 --- /dev/null +++ b/tools/hive/kerberos/templates/sentry-site.xml.hive-client.template @@ -0,0 +1,44 @@ + + + sentry.hive.provider + org.apache.sentry.provider.file.HadoopGroupResourceAuthorizationProvider + + + sentry.hive.server + server1 + + + sentry.hive.testing.mode + false + + + sentry.service.client.server.rpc-port + 8038 + + + sentry.service.client.server.rpc-address + localhost + + + sentry.service.client.server.rpc-connection-timeout + 200000 + + + + + sentry.hive.provider.backend + org.apache.sentry.provider.db.SimpleDBProviderBackend + + + sentry.service.security.mode + kerberos + + + sentry.service.server.principal + sentry/{{HOSTNAME}}@LOCAL + + + sentry.metastore.service.users + hive + + diff --git a/tools/hive/kerberos/templates/sentry-site.xml.server.template b/tools/hive/kerberos/templates/sentry-site.xml.server.template new file mode 100644 index 00000000..096e2d9c --- /dev/null +++ b/tools/hive/kerberos/templates/sentry-site.xml.server.template @@ -0,0 +1,50 @@ + + + sentry.hive.server + server1 + + + sentry.store.jdbc.url + jdbc:derby:;databaseName=metastore_db;create=true + + + sentry.service.security.mode + kerberos + + + sentry.service.server.principal + sentry/{{HOSTNAME}}@LOCAL + + + sentry.service.server.keytab + /usr/local/hadoop/etc/hadoop/hdfs.keytab + + + sentry.service.admin.group + hdfs,hive + + + sentry.service.allow.connect + hive + + + sentry.store.jdbc.driver + org.apache.derby.jdbc.EmbeddedDriver + + + sentry.store.jdbc.user + sentry + + + sentry.store.jdbc.user + sentry + + + sentry.store.jdbc.password + test + + + sentry.verify.schema.version + true + + From c20fcc7c1604ac12f16a5f891914a881601e0d1e Mon Sep 17 00:00:00 2001 From: "Susan X. Huynh" Date: Mon, 27 Aug 2018 15:10:42 -0700 Subject: [PATCH 02/10] moved sentry stuff into kerberos directory --- tools/hive/hadoop-hive/Dockerfile | 10 ----- .../templates/hive-site.xml.template | 37 ----------------- tools/hive/kerberos/Dockerfile | 11 ++++- .../templates/hive-site-kerberos.xml.template | 40 +++++++++++++++++-- 4 files changed, 46 insertions(+), 52 deletions(-) diff --git a/tools/hive/hadoop-hive/Dockerfile b/tools/hive/hadoop-hive/Dockerfile index d3ef1c9b..c7a97d53 100644 --- a/tools/hive/hadoop-hive/Dockerfile +++ b/tools/hive/hadoop-hive/Dockerfile @@ -17,7 +17,6 @@ ENV POSTGRESQL_MAIN /var/lib/postgresql/9.5/main/ ENV POSTGRESQL_CONFIG_FILE /var/lib/postgresql/9.5/main/postgresql.conf ENV POSTGRESQL_BIN /usr/lib/postgresql/9.5/bin/postgres ENV PGPASSWORD hive -ENV SENTRY_HOME /usr/local/sentry # install dev tools RUN apt-get update && \ @@ -146,14 +145,5 @@ RUN chmod 700 /etc/hive-bootstrap.sh EXPOSE 10000 10001 10002 10003 9083 50111 5432 -# sentry -RUN curl -L http://archive.cloudera.com/cdh5/cdh/5/sentry-1.5.1-cdh5.11.0.tar.gz | tar -xzC /usr/local && \ - cd /usr/local && \ - ln -s apache-sentry-1.5.1-cdh5.11.0-bin/ sentry -ADD templates/sentry-site.xml.hive-client.template /usr/local/hive/conf/sentry-site.xml.template -ADD templates/sentry-site.xml.server.template /usr/local/sentry/conf/sentry-site.xml.template -ADD scripts/test-hive-sentry.sh /etc/test-hive-sentry.sh -RUN chmod 700 /etc/test-hive-sentry.sh - # run bootstrap script CMD ["/etc/hive-bootstrap.sh", "-d"] diff --git a/tools/hive/hadoop-hive/templates/hive-site.xml.template b/tools/hive/hadoop-hive/templates/hive-site.xml.template index 6127d942..8e61b910 100755 --- a/tools/hive/hadoop-hive/templates/hive-site.xml.template +++ b/tools/hive/hadoop-hive/templates/hive-site.xml.template @@ -20,41 +20,4 @@ hive.metastore.warehouse.dir /apps/hive/warehouse - - - - hive.server2.enable.doAs - false - - - - hive.security.authorization.task.factory - org.apache.sentry.binding.hive.SentryHiveAuthorizationTaskFactoryImpl - - - - hive.server2.session.hook - org.apache.sentry.binding.hive.HiveAuthzBindingSessionHook - - - - hive.sentry.conf.url - file:///usr/local/hive/conf/sentry-site.xml - - - - - hive.metastore.filter.hook - org.apache.sentry.binding.metastore.SentryMetaStoreFilterHook - - - - hive.metastore.pre.event.listeners - org.apache.sentry.binding.metastore.MetastoreAuthzBinding - - - - hive.metastore.event.listeners - org.apache.sentry.binding.metastore.SentryMetastorePostEventListener - diff --git a/tools/hive/kerberos/Dockerfile b/tools/hive/kerberos/Dockerfile index d2f84c34..ba2a8577 100644 --- a/tools/hive/kerberos/Dockerfile +++ b/tools/hive/kerberos/Dockerfile @@ -1,5 +1,14 @@ FROM cdh5-hive +ENV SENTRY_VERSION 1.5.1 +ENV SENTRY_HOME /usr/local/sentry + +# download sentry +RUN curl -L http://archive.cloudera.com/cdh${CDH_VERSION}/cdh/${CDH_VERSION}/sentry-${SENTRY_VERSION}-cdh${CDH_EXACT_VERSION}.tar.gz \ + | tar -xzC /usr/local && \ + cd /usr/local && \ + ln -s apache-sentry-${SENTRY_VERSION}-cdh${CDH_EXACT_VERSION}-bin/ sentry + # copy kerberized hadoop config files ADD templates/core-site.xml.template $HADOOP_PREFIX/etc/hadoop/core-site.xml.template ADD templates/hdfs-site.xml.template $HADOOP_PREFIX/etc/hadoop/hdfs-site.xml.template @@ -9,7 +18,7 @@ ADD templates/yarn-site.xml.template $HADOOP_PREFIX/etc/hadoop/yarn-site.xml.tem ADD templates/hive-site.xml.template /opt/files/ ADD templates/hive-site.xml.template $HIVE_CONF/hive-site.xml.template -# kerberized sentry config files +# sentry config files ADD templates/sentry-site.xml.hive-client.template /usr/local/hive/conf/sentry-site.xml.template ADD templates/sentry-site.xml.server.template /usr/local/sentry/conf/sentry-site.xml.template diff --git a/tools/hive/kerberos/templates/hive-site-kerberos.xml.template b/tools/hive/kerberos/templates/hive-site-kerberos.xml.template index a500c6c2..d3a3a08b 100755 --- a/tools/hive/kerberos/templates/hive-site-kerberos.xml.template +++ b/tools/hive/kerberos/templates/hive-site-kerberos.xml.template @@ -41,8 +41,40 @@ hive - + + + hive.server2.enable.doAs + false + + + + hive.security.authorization.task.factory + org.apache.sentry.binding.hive.SentryHiveAuthorizationTaskFactoryImpl + + + + hive.server2.session.hook + org.apache.sentry.binding.hive.HiveAuthzBindingSessionHook + + + + hive.sentry.conf.url + file:///usr/local/hive/conf/sentry-site.xml + + + + + hive.metastore.filter.hook + org.apache.sentry.binding.metastore.SentryMetaStoreFilterHook + + + + hive.metastore.pre.event.listeners + org.apache.sentry.binding.metastore.MetastoreAuthzBinding + + + + hive.metastore.event.listeners + org.apache.sentry.binding.metastore.SentryMetastorePostEventListener + From 05349aa7fbe762a01723b81f115da6fae807c6c5 Mon Sep 17 00:00:00 2001 From: "Susan X. Huynh" Date: Tue, 28 Aug 2018 09:02:27 -0700 Subject: [PATCH 03/10] Updated README, removed unnecessary config properties --- tools/hive/README.md | 7 +++++-- .../hive/hadoop-hive/scripts/hive-bootstrap.sh | 11 +++++++---- .../kerberos/marathon/hdfs-hive-kerberos.json | 2 +- .../hive/kerberos/scripts/test-hive-sentry.sh | 10 ---------- .../templates/hive-site-kerberos.xml.template | 5 ----- .../sentry-site.xml.hive-client.template | 12 ------------ .../templates/sentry-site.xml.server.template | 18 +----------------- 7 files changed, 14 insertions(+), 51 deletions(-) diff --git a/tools/hive/README.md b/tools/hive/README.md index 97d816f1..e6cb0d6a 100644 --- a/tools/hive/README.md +++ b/tools/hive/README.md @@ -1,7 +1,10 @@ -# Cloudera Hadoop and Hive Docker Image with Kerberos +# Cloudera Hadoop and Hive Docker Image with Kerberos, Sentry -This is a Hadoop Docker image running CDH5 versions of Hadoop and Hive, all in one container. There is a separate Kerberos image in which Hadoop and Hive use Kerberos for authentication. Adapted from https://github.com/tilakpatidar/cdh5_hive_postgres and based on Ubuntu (trusty). +This is a Hadoop Docker image running CDH5 versions of Hadoop and Hive, all in one container. +There is a separate Kerberos image in which Hadoop and Hive use Kerberos for authentication, +and Sentry for authorization. +Adapted from https://github.com/tilakpatidar/cdh5_hive_postgres and based on Ubuntu (trusty). Postgres is also installed so that Hive can use it for its Metastore backend and run in remote mode. diff --git a/tools/hive/hadoop-hive/scripts/hive-bootstrap.sh b/tools/hive/hadoop-hive/scripts/hive-bootstrap.sh index 126d5fe0..05b3f4c3 100644 --- a/tools/hive/hadoop-hive/scripts/hive-bootstrap.sh +++ b/tools/hive/hadoop-hive/scripts/hive-bootstrap.sh @@ -7,10 +7,14 @@ printenv | cat >> /root/.bashrc /etc/hadoop-bootstrap.sh -d # init and start sentry +SENTRY_CONF_TEMPLATE=$SENTRY_HOME/conf/sentry-site.xml.template SENTRY_CONF_FILE=$SENTRY_HOME/conf/sentry-site.xml -sed s/{{HOSTNAME}}/$HOSTNAME/ $SENTRY_HOME/conf/sentry-site.xml.template > $SENTRY_HOME/conf/sentry-site.xml -$SENTRY_HOME/bin/sentry --command schema-tool --conffile $SENTRY_CONF_FILE --dbType derby --initSchema -$SENTRY_HOME/bin/sentry --command service --conffile $SENTRY_CONF_FILE & +if [ -f "$SENTRY_CONF_TEMPLATE" ]; then + sed s/{{HOSTNAME}}/$HOSTNAME/ $SENTRY_HOME/conf/sentry-site.xml.template > $SENTRY_HOME/conf/sentry-site.xml + sed s/{{HOSTNAME}}/$HOSTNAME/ $HIVE_CONF/sentry-site.xml.template > $HIVE_CONF/sentry-site.xml + $SENTRY_HOME/bin/sentry --command schema-tool --conffile $SENTRY_CONF_FILE --dbType derby --initSchema + $SENTRY_HOME/bin/sentry --command service --conffile $SENTRY_CONF_FILE & +fi # restart postgresql /etc/init.d/postgresql restart @@ -41,7 +45,6 @@ hdfs dfs -chmod 777 /tmp/hive # altering the hive-site configuration sed s/{{HOSTNAME}}/$HOSTNAME/ $HIVE_CONF/hive-site.xml.template > $HIVE_CONF/hive-site.xml -sed s/{{HOSTNAME}}/$HOSTNAME/ $HIVE_CONF/sentry-site.xml.template > $HIVE_CONF/sentry-site.xml sed s/{{HOSTNAME}}/$HOSTNAME/ /opt/files/hive-site.xml.template > /opt/files/hive-site.xml # start hive metastore server diff --git a/tools/hive/kerberos/marathon/hdfs-hive-kerberos.json b/tools/hive/kerberos/marathon/hdfs-hive-kerberos.json index 1360bb0b..93d25e78 100644 --- a/tools/hive/kerberos/marathon/hdfs-hive-kerberos.json +++ b/tools/hive/kerberos/marathon/hdfs-hive-kerberos.json @@ -33,7 +33,7 @@ [ "hostname", "IS", - "10.0.1.160" + "10.0.1.24" ] ] } diff --git a/tools/hive/kerberos/scripts/test-hive-sentry.sh b/tools/hive/kerberos/scripts/test-hive-sentry.sh index f5e0dd89..682eda73 100644 --- a/tools/hive/kerberos/scripts/test-hive-sentry.sh +++ b/tools/hive/kerberos/scripts/test-hive-sentry.sh @@ -16,16 +16,6 @@ GRANT ALL on DATABASE default to ROLE test_role WITH GRANT OPTION; EOF beeline -u "jdbc:hive2://localhost:10000/default;principal=hive/${HOSTNAME}@LOCAL" -f grant_alice.sql -# Test Hive / Sentry -#echo "Create a table in Hive with Sentry as alice ..." -#kdestroy -#kinit -k -t /usr/local/hadoop/etc/hadoop/hdfs.keytab alice/${HOSTNAME}@LOCAL -#cat <create_table.sql -#CREATE TABLE test1 (col1 INT); -#SHOW TABLES; -#EOF -#beeline -u "jdbc:hive2://localhost:10000/default;principal=alice/${HOSTNAME}@LOCAL" -f create_table.sql - # Log back in as hdfs kdestroy kinit -k -t /usr/local/hadoop/etc/hadoop/hdfs.keytab hdfs@LOCAL diff --git a/tools/hive/kerberos/templates/hive-site-kerberos.xml.template b/tools/hive/kerberos/templates/hive-site-kerberos.xml.template index d3a3a08b..c25d5d52 100755 --- a/tools/hive/kerberos/templates/hive-site-kerberos.xml.template +++ b/tools/hive/kerberos/templates/hive-site-kerberos.xml.template @@ -57,11 +57,6 @@ org.apache.sentry.binding.hive.HiveAuthzBindingSessionHook - - hive.sentry.conf.url - file:///usr/local/hive/conf/sentry-site.xml - - hive.metastore.filter.hook diff --git a/tools/hive/kerberos/templates/sentry-site.xml.hive-client.template b/tools/hive/kerberos/templates/sentry-site.xml.hive-client.template index 393d0118..92a8cda2 100644 --- a/tools/hive/kerberos/templates/sentry-site.xml.hive-client.template +++ b/tools/hive/kerberos/templates/sentry-site.xml.hive-client.template @@ -7,10 +7,6 @@ sentry.hive.server server1 - - sentry.hive.testing.mode - false - sentry.service.client.server.rpc-port 8038 @@ -19,20 +15,12 @@ sentry.service.client.server.rpc-address localhost - - sentry.service.client.server.rpc-connection-timeout - 200000 - sentry.hive.provider.backend org.apache.sentry.provider.db.SimpleDBProviderBackend - - sentry.service.security.mode - kerberos - sentry.service.server.principal sentry/{{HOSTNAME}}@LOCAL diff --git a/tools/hive/kerberos/templates/sentry-site.xml.server.template b/tools/hive/kerberos/templates/sentry-site.xml.server.template index 096e2d9c..9c9c8f6f 100644 --- a/tools/hive/kerberos/templates/sentry-site.xml.server.template +++ b/tools/hive/kerberos/templates/sentry-site.xml.server.template @@ -7,10 +7,6 @@ sentry.store.jdbc.url jdbc:derby:;databaseName=metastore_db;create=true - - sentry.service.security.mode - kerberos - sentry.service.server.principal sentry/{{HOSTNAME}}@LOCAL @@ -21,20 +17,12 @@ sentry.service.admin.group - hdfs,hive + hive sentry.service.allow.connect hive - - sentry.store.jdbc.driver - org.apache.derby.jdbc.EmbeddedDriver - - - sentry.store.jdbc.user - sentry - sentry.store.jdbc.user sentry @@ -43,8 +31,4 @@ sentry.store.jdbc.password test - - sentry.verify.schema.version - true - From ccbd2a7dd1fc7be05db3944ff5464a61eed359d2 Mon Sep 17 00:00:00 2001 From: "Susan X. Huynh" Date: Tue, 28 Aug 2018 09:14:59 -0700 Subject: [PATCH 04/10] Removed the test script. --- .../hive/kerberos/scripts/test-hive-sentry.sh | 21 ------------------- 1 file changed, 21 deletions(-) delete mode 100644 tools/hive/kerberos/scripts/test-hive-sentry.sh diff --git a/tools/hive/kerberos/scripts/test-hive-sentry.sh b/tools/hive/kerberos/scripts/test-hive-sentry.sh deleted file mode 100644 index 682eda73..00000000 --- a/tools/hive/kerberos/scripts/test-hive-sentry.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -set -x - -# Create a user "alice" since Sentry authorization relies on the Linux user and group information -useradd alice - -# Grant permissions to user “alice” -echo "Grant permissions to user alice ..." -kdestroy -kinit -k -t /usr/local/hadoop/etc/hadoop/hdfs.keytab hive/${HOSTNAME}@LOCAL -cat <grant_alice.sql -CREATE ROLE test_role; -GRANT ROLE test_role to GROUP alice; -GRANT ROLE test_role to GROUP root; -GRANT ALL on DATABASE default to ROLE test_role WITH GRANT OPTION; -EOF -beeline -u "jdbc:hive2://localhost:10000/default;principal=hive/${HOSTNAME}@LOCAL" -f grant_alice.sql - -# Log back in as hdfs -kdestroy -kinit -k -t /usr/local/hadoop/etc/hadoop/hdfs.keytab hdfs@LOCAL From 0ff9e966489d56b3cb5769f8fabb1fa7d9fdb931 Mon Sep 17 00:00:00 2001 From: "Susan X. Huynh" Date: Tue, 28 Aug 2018 09:56:09 -0700 Subject: [PATCH 05/10] Remove test script from Dockerfile, too --- tools/hive/kerberos/Dockerfile | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tools/hive/kerberos/Dockerfile b/tools/hive/kerberos/Dockerfile index ba2a8577..e7065169 100644 --- a/tools/hive/kerberos/Dockerfile +++ b/tools/hive/kerberos/Dockerfile @@ -22,10 +22,6 @@ ADD templates/hive-site.xml.template $HIVE_CONF/hive-site.xml.template ADD templates/sentry-site.xml.hive-client.template /usr/local/hive/conf/sentry-site.xml.template ADD templates/sentry-site.xml.server.template /usr/local/sentry/conf/sentry-site.xml.template -# hive / sentry test script -ADD scripts/test-hive-sentry.sh /etc/test-hive-sentry.sh -RUN chmod 700 /etc/test-hive-sentry.sh - # krb5.conf ADD conf/krb5.conf /etc/ From fa014d0ba8f390083acfb296734861bd2b156530 Mon Sep 17 00:00:00 2001 From: "Susan X. Huynh" Date: Thu, 30 Aug 2018 08:54:17 -0700 Subject: [PATCH 06/10] Revert "Remove test script from Dockerfile, too" This reverts commit 0ff9e966489d56b3cb5769f8fabb1fa7d9fdb931. --- tools/hive/kerberos/Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/hive/kerberos/Dockerfile b/tools/hive/kerberos/Dockerfile index e7065169..ba2a8577 100644 --- a/tools/hive/kerberos/Dockerfile +++ b/tools/hive/kerberos/Dockerfile @@ -22,6 +22,10 @@ ADD templates/hive-site.xml.template $HIVE_CONF/hive-site.xml.template ADD templates/sentry-site.xml.hive-client.template /usr/local/hive/conf/sentry-site.xml.template ADD templates/sentry-site.xml.server.template /usr/local/sentry/conf/sentry-site.xml.template +# hive / sentry test script +ADD scripts/test-hive-sentry.sh /etc/test-hive-sentry.sh +RUN chmod 700 /etc/test-hive-sentry.sh + # krb5.conf ADD conf/krb5.conf /etc/ From a78673706c47711c242a87731a1663b886b18394 Mon Sep 17 00:00:00 2001 From: "Susan X. Huynh" Date: Thu, 30 Aug 2018 08:54:39 -0700 Subject: [PATCH 07/10] Revert "Removed the test script." This reverts commit ccbd2a7dd1fc7be05db3944ff5464a61eed359d2. --- .../hive/kerberos/scripts/test-hive-sentry.sh | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 tools/hive/kerberos/scripts/test-hive-sentry.sh diff --git a/tools/hive/kerberos/scripts/test-hive-sentry.sh b/tools/hive/kerberos/scripts/test-hive-sentry.sh new file mode 100644 index 00000000..682eda73 --- /dev/null +++ b/tools/hive/kerberos/scripts/test-hive-sentry.sh @@ -0,0 +1,21 @@ +#!/bin/bash +set -x + +# Create a user "alice" since Sentry authorization relies on the Linux user and group information +useradd alice + +# Grant permissions to user “alice” +echo "Grant permissions to user alice ..." +kdestroy +kinit -k -t /usr/local/hadoop/etc/hadoop/hdfs.keytab hive/${HOSTNAME}@LOCAL +cat <grant_alice.sql +CREATE ROLE test_role; +GRANT ROLE test_role to GROUP alice; +GRANT ROLE test_role to GROUP root; +GRANT ALL on DATABASE default to ROLE test_role WITH GRANT OPTION; +EOF +beeline -u "jdbc:hive2://localhost:10000/default;principal=hive/${HOSTNAME}@LOCAL" -f grant_alice.sql + +# Log back in as hdfs +kdestroy +kinit -k -t /usr/local/hadoop/etc/hadoop/hdfs.keytab hdfs@LOCAL From c16194b16b67299d824cdfbaed90c75c37006c1e Mon Sep 17 00:00:00 2001 From: "Susan X. Huynh" Date: Thu, 30 Aug 2018 09:00:19 -0700 Subject: [PATCH 08/10] fixed grant script to work from "dcos task exec" --- .../{test-hive-sentry.sh => grant-hive-privileges.sh} | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) rename tools/hive/kerberos/scripts/{test-hive-sentry.sh => grant-hive-privileges.sh} (75%) diff --git a/tools/hive/kerberos/scripts/test-hive-sentry.sh b/tools/hive/kerberos/scripts/grant-hive-privileges.sh similarity index 75% rename from tools/hive/kerberos/scripts/test-hive-sentry.sh rename to tools/hive/kerberos/scripts/grant-hive-privileges.sh index 682eda73..3e442b33 100644 --- a/tools/hive/kerberos/scripts/test-hive-sentry.sh +++ b/tools/hive/kerberos/scripts/grant-hive-privileges.sh @@ -1,8 +1,10 @@ #!/bin/bash set -x +export HADOOP_HOME=/usr/local/hadoop + # Create a user "alice" since Sentry authorization relies on the Linux user and group information -useradd alice +/usr/sbin/useradd alice # Grant permissions to user “alice” echo "Grant permissions to user alice ..." @@ -14,7 +16,7 @@ GRANT ROLE test_role to GROUP alice; GRANT ROLE test_role to GROUP root; GRANT ALL on DATABASE default to ROLE test_role WITH GRANT OPTION; EOF -beeline -u "jdbc:hive2://localhost:10000/default;principal=hive/${HOSTNAME}@LOCAL" -f grant_alice.sql +/usr/local/hive/bin/beeline -u "jdbc:hive2://localhost:10000/default;principal=hive/${HOSTNAME}@LOCAL" -f grant_alice.sql # Log back in as hdfs kdestroy From f6ab334850dcc4a5e003b1f50ad014be4949b0e0 Mon Sep 17 00:00:00 2001 From: "Susan X. Huynh" Date: Thu, 30 Aug 2018 09:03:28 -0700 Subject: [PATCH 09/10] fixed Dockerfile with new script name --- tools/hive/kerberos/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/hive/kerberos/Dockerfile b/tools/hive/kerberos/Dockerfile index ba2a8577..8fa96b69 100644 --- a/tools/hive/kerberos/Dockerfile +++ b/tools/hive/kerberos/Dockerfile @@ -23,8 +23,8 @@ ADD templates/sentry-site.xml.hive-client.template /usr/local/hive/conf/sentry-s ADD templates/sentry-site.xml.server.template /usr/local/sentry/conf/sentry-site.xml.template # hive / sentry test script -ADD scripts/test-hive-sentry.sh /etc/test-hive-sentry.sh -RUN chmod 700 /etc/test-hive-sentry.sh +ADD scripts/grant-hive-privileges.sh /etc/grant-hive-privileges.sh +RUN chmod 700 /etc/grant-hive-privileges.sh # krb5.conf ADD conf/krb5.conf /etc/ From bba9a8da32ae08826d71d1032ac6ad13f8384d94 Mon Sep 17 00:00:00 2001 From: Sam Tran Date: Wed, 19 Sep 2018 12:27:58 -0700 Subject: [PATCH 10/10] Make app.json host IP constraint look more like replaceable value IP value gets replaced in the python integration test --- tools/hive/kerberos/marathon/hdfs-hive-kerberos.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/hive/kerberos/marathon/hdfs-hive-kerberos.json b/tools/hive/kerberos/marathon/hdfs-hive-kerberos.json index 93d25e78..e1650a04 100644 --- a/tools/hive/kerberos/marathon/hdfs-hive-kerberos.json +++ b/tools/hive/kerberos/marathon/hdfs-hive-kerberos.json @@ -33,7 +33,7 @@ [ "hostname", "IS", - "10.0.1.24" + "1.2.3.4" ] ] }