apache · kumaab · Aug 26, 2024 · Aug 30, 2024 · Jan 24, 2025 · Jan 24, 2025
diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml
@@ -140,6 +140,7 @@ jobs:
           -f docker-compose.ranger-kafka.yml \
           -f docker-compose.ranger-hive.yml \
           -f docker-compose.ranger-knox.yml \
+          -f docker-compose.ranger-impala.yml \
           -f docker-compose.ranger-ozone.yml build
 
       - name: Bring up containers
@@ -158,11 +159,13 @@ jobs:
           -f docker-compose.ranger-kafka.yml \
           -f docker-compose.ranger-hive.yml \
           -f docker-compose.ranger-knox.yml \
+          -f docker-compose.ranger-impala.yml \
           -f docker-compose.ranger-ozone.yml up -d
+
       - name: Check status of containers and remove them
         run: | 
           sleep 60
-          containers=(ranger ranger-zk ranger-solr ranger-postgres ranger-usersync ranger-tagsync ranger-kms ranger-hadoop ranger-hbase ranger-kafka ranger-hive ranger-knox ozone-om ozone-scm ozone-datanode);
+          containers=(ranger ranger-zk ranger-solr ranger-postgres ranger-usersync ranger-tagsync ranger-kms ranger-hadoop ranger-hbase ranger-kafka ranger-hive ranger-knox ozone-om ozone-scm ozone-datanode impalad impala-statestored impala-catalogd);
           flag=true;
           for container in "${containers[@]}"; do
               if [[ $(docker inspect -f '{{.State.Running}}' $container 2>/dev/null) == "true" ]]; then
@@ -180,3 +183,8 @@ jobs:
               docker stop $(docker ps -q) && docker rm $(docker ps -aq);
               exit 1;
           fi
+
+      - name: Check impalad logs - testing # to be removed
+        run: |
+          sleep 60
+          docker logs impalad
diff --git a/dev-support/ranger-docker/.env b/dev-support/ranger-docker/.env
@@ -53,6 +53,8 @@ OZONE_RUNNER_VERSION=20230615-1
 OZONE_RUNNER_IMAGE=apache/ozone-runner
 OZONE_OPTS=
 
+IMPALA_VERSION=4.0.0
+
 # versions of ranger services
 RANGER_VERSION=3.0.0-SNAPSHOT
 KMS_VERSION=3.0.0-SNAPSHOT

diff --git a/dev-support/ranger-docker/Dockerfile.impalad b/dev-support/ranger-docker/Dockerfile.impalad
@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG IMPALA_VERSION
+FROM apache/impala:${IMPALA_VERSION}-impalad_coord_exec
+
+USER root
+
+RUN mkdir -p /var/lib/ranger/impala
+RUN mkdir -p -m 755 /var/log/impala
+RUN chown impala:impala /var/lib/ranger/impala
+RUN chown impala:impala -R /var/log/impala
+
+# Install impala-shell and its dependencies
+RUN apt-get update
+RUN apt-get install -y libkrb5-dev
+RUN apt-get install -y python3-dev python3-pip python3
+RUN apt-get install -y gcc g++
+RUN apt-get install -y libsasl2-dev libssl-dev
+RUN apt-get clean && rm -rf /var/lib/apt/lists/*
+
+RUN pip3 install impala-shell
+USER impala
+
+ENTRYPOINT ["/opt/impala/bin/daemon_entrypoint.sh", "/opt/impala/bin/impalad",\
+     "-log_dir=/opt/impala/logs",\
+     "-abort_on_config_error=false", "-state_store_host=impala-statestored",\
+     "-catalog_service_host=impala-catalogd", "-mem_limit_includes_jvm=true",\
+     "-use_local_catalog=true", "--rpc_use_loopback=true",\
+     "-use_resolved_hostname=true",\
+     "--ranger_service_type=hive",\
+     "--ranger_app_id=impala",\
+     "--authorization_provider=ranger",\
+     ""]
diff --git a/dev-support/ranger-docker/Dockerfile.ranger-base b/dev-support/ranger-docker/Dockerfile.ranger-base
@@ -50,6 +50,7 @@ RUN groupadd ranger && \
     useradd -g hadoop -ms /bin/bash hive && \
     useradd -g hadoop -ms /bin/bash hbase && \
     useradd -g hadoop -ms /bin/bash kafka && \
+    useradd -g hadoop -ms /bin/bash impala && \
     useradd -g hadoop -ms /bin/bash ozone && \
     groupadd knox && \
     useradd -g knox -ms /bin/bash knox && \

diff --git a/dev-support/ranger-docker/Dockerfile.ranger-hive b/dev-support/ranger-docker/Dockerfile.ranger-hive
@@ -53,5 +53,6 @@ ENV HIVE_HOME=/opt/hive
 ENV HADOOP_HOME=/opt/hadoop
 ENV PATH=/usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/hive/bin:/opt/hadoop/bin
 
+COPY ./config/hive/hive-log4j2.properties               ${HIVE_HOME}/hive-log4j2.properties
 
 ENTRYPOINT [ "/home/ranger/scripts/ranger-hive.sh" ]
diff --git a/dev-support/ranger-docker/config/hive/hive-log4j2.properties b/dev-support/ranger-docker/config/hive/hive-log4j2.properties
@@ -0,0 +1,83 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+status = INFO
+name = HiveLog4j2
+packages = org.apache.hadoop.hive.ql.log
+
+# list of properties
+property.hive.log.level = INFO
+property.hive.root.logger = DRFA
+property.hive.log.dir = /opt/hive
+property.hive.log.file = hive.log
+property.hive.perflogger.log.level = INFO
+
+# list of all appenders
+appenders = console, DRFA
+
+# console appender
+appender.console.type = Console
+appender.console.name = console
+appender.console.target = SYSTEM_ERR
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = %d{ISO8601} %5p [%t] %c{2}: %m%n
+
+# daily rolling file appender
+appender.DRFA.type = RollingRandomAccessFile
+appender.DRFA.name = DRFA
+appender.DRFA.fileName = ${sys:hive.log.dir}/${sys:hive.log.file}
+# Use %pid in the filePattern to append <process-id>@<host-name> to the filename if you want separate log files for different CLI session
+appender.DRFA.filePattern = ${sys:hive.log.dir}/${sys:hive.log.file}.%d{yyyy-MM-dd}
+appender.DRFA.layout.type = PatternLayout
+appender.DRFA.layout.pattern = %d{ISO8601} %5p [%t] %c{2}: %m%n
+appender.DRFA.policies.type = Policies
+appender.DRFA.policies.time.type = TimeBasedTriggeringPolicy
+appender.DRFA.policies.time.interval = 1
+appender.DRFA.policies.time.modulate = true
+appender.DRFA.strategy.type = DefaultRolloverStrategy
+appender.DRFA.strategy.max = 30
+
+# list of all loggers
+loggers = NIOServerCnxn, ClientCnxnSocketNIO, DataNucleus, Datastore, JPOX, PerfLogger, AmazonAws, ApacheHttp
+
+logger.NIOServerCnxn.name = org.apache.zookeeper.server.NIOServerCnxn
+logger.NIOServerCnxn.level = WARN
+
+logger.ClientCnxnSocketNIO.name = org.apache.zookeeper.ClientCnxnSocketNIO
+logger.ClientCnxnSocketNIO.level = WARN
+
+logger.DataNucleus.name = DataNucleus
+logger.DataNucleus.level = ERROR
+
+logger.Datastore.name = Datastore
+logger.Datastore.level = ERROR
+
+logger.JPOX.name = JPOX
+logger.JPOX.level = ERROR
+
+logger.AmazonAws.name=com.amazonaws
+logger.AmazonAws.level = INFO
+
+logger.ApacheHttp.name=org.apache.http
+logger.ApacheHttp.level = INFO
+
+logger.PerfLogger.name = org.apache.hadoop.hive.ql.log.PerfLogger
+logger.PerfLogger.level = ${sys:hive.perflogger.log.level}
+
+# root logger
+rootLogger.level = ${sys:hive.log.level}
+rootLogger.appenderRefs = root
+rootLogger.appenderRef.root.ref = ${sys:hive.root.logger}
diff --git a/dev-support/ranger-docker/config/impala/hive-site.xml b/dev-support/ranger-docker/config/impala/hive-site.xml
@@ -0,0 +1,30 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?><!--
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+    <property>
+        <name>hive.metastore.uris</name>
+        <value>thrift://ranger-hive:9083</value>
+    </property>
+    <property>
+        <name>hive.metastore.event.db.notification.api.auth</name>
+        <value>false</value>
+    </property>
+</configuration>
diff --git a/dev-support/ranger-docker/config/impala/ranger-hive-audit.xml b/dev-support/ranger-docker/config/impala/ranger-hive-audit.xml
@@ -0,0 +1,78 @@
+<configuration>
+    <property>
+        <name>xasecure.audit.is.enabled</name>
+        <value>true</value>
+    </property>
+    <property>
+        <name>xasecure.audit.provider.summary.enabled</name>
+        <value>true</value>
+    </property>
+    <property>
+        <name>xasecure.audit.destination.hdfs</name>
+        <value>true</value>
+    </property>
+    <property>
+        <name>xasecure.audit.destination.hdfs.dir</name>
+        <value>hdfs://ranger-hadoop:9000/ranger/audit</value>
+    </property>
+    <property>
+        <name>xasecure.audit.destination.hdfs.batch.filespool.dir</name>
+        <value>/var/log/impala/audit/hdfs/spool</value>
+    </property>
+    <property>
+        <name>xasecure.audit.destination.solr</name>
+        <value>true</value>
+    </property>
+    <property>
+        <name>xasecure.audit.destination.solr.urls</name>
+        <value>http://ranger-solr:8983/solr/ranger_audits</value>
+    </property>
+    <property>
+        <name>xasecure.audit.destination.solr.zookeepers</name>
+        <value></value>
+    </property>
+    <property>
+        <name>xasecure.audit.solr.solr_url</name>
+        <value>http://ranger-solr:8983/solr/ranger_audits</value>
+    </property>
+    <property>
+        <name>xasecure.audit.destination.solr.batch.filespool.dir</name>
+        <value>/var/log/impala/audit/solr/spool</value>
+    </property>
+    <property>
+        <name>xasecure.audit.jaas.Client.loginModuleName</name>
+        <value></value>
+    </property>
+    <property>
+        <name>xasecure.audit.jaas.Client.loginModuleControlFlag</name>
+        <value>required</value>
+    </property>
+    <property>
+        <name>xasecure.audit.jaas.Client.option.useKeyTab</name>
+        <value>false</value>
+    </property>
+    <property>
+        <name>xasecure.audit.jaas.Client.option.storeKey</name>
+        <value>false</value>
+    </property>
+    <property>
+        <name>xasecure.audit.jaas.Client.option.serviceName</name>
+        <value>solr</value>
+    </property>
+    <property>
+        <name>xasecure.audit.destination.solr.force.use.inmemory.jaas.config</name>
+        <value>true</value>
+    </property>
+    <property>
+        <name>xasecure.audit.jaas.Client.option.principal</name>
+        <value></value>
+    </property>
+    <property>
+        <name>xasecure.audit.jaas.Client.option.keyTab</name>
+        <value></value>
+    </property>
+    <property>
+        <name>xasecure.audit.destination.metrics</name>
+        <value>false</value>
+    </property>
+</configuration>
diff --git a/dev-support/ranger-docker/config/impala/ranger-hive-security.xml b/dev-support/ranger-docker/config/impala/ranger-hive-security.xml
@@ -0,0 +1,79 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+      http://www.apache.org/licenses/LICENSE-2.0
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<configuration>
+    <property>
+        <name>ranger.plugin.hive.service.name</name>
+        <value>dev_hive</value>
+        <description>Name of the Ranger service containing policies.</description>
+    </property>
+    <property>
+        <name>ranger.plugin.hive.policy.rest.url</name>
+        <value>http://ranger:6080</value>
+        <description>URL to Ranger Admin.</description>
+    </property>
+    <property>
+        <name>ranger.plugin.hive.policy.cache.dir</name>
+        <value>/var/lib/ranger/impala/policy-cache</value>
+        <description>
+            Directory where Ranger policies are cached after successful retrieval from the source.
+        </description>
+    </property>
+    <property>
+        <name>ranger.plugin.hive.policy.source.impl</name>
+        <value>org.apache.ranger.admin.client.RangerAdminRESTClient</value>
+    </property>
+    <property>
+        <name>ranger.plugin.hive.policy.pollIntervalMs</name>
+        <value>30000</value>
+        <description>
+            Polling interval in milliseconds to poll for changes in policies.
+        </description>
+    </property>
+    <property>
+        <name>ranger.plugin.hive.policy.rest.client.connection.timeoutMs</name>
+        <value>120000</value>
+        <description>
+            RangerRestClient connection timeout in milliseconds.
+        </description>
+    </property>
+    <property>
+        <name>ranger.plugin.hive.policy.rest.client.read.timeoutMs</name>
+        <value>30000</value>
+        <description>
+            RangerRestClient read timeout in milliseconds.
+        </description>
+    </property>
+    <property>
+        <name>ranger.plugin.hive.disable.cache.if.servicenotfound</name>
+        <value>false</value>
+    </property>
+    <property>
+        <name>ranger.plugin.hive.use.x-forwarded-for.ipaddress</name>
+        <value>false</value>
+    </property>
+    <property>
+        <name>ranger.plugin.hive.trusted.proxy.ipaddress</name>
+        <value></value>
+    </property>
+    <property>
+        <name>xasecure.hive.update.xapolicies.on.grant.revoke</name>
+        <value>true</value>
+    </property>
+    <property>
+        <name>ranger.plugin.hive.urlauth.filesystem.schemes</name>
+        <value>hdfs:,file:,wasb:,adl:</value>
+    </property>
+</configuration>