diff --git a/Dockerfile b/Dockerfile index cf2b8f382f..ec34b5d22b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,7 +34,6 @@ RUN apt-get update \ netcat \ curl \ && cd / \ - && mkdir /opt/ranger-atlas-plugin \ && export MAVEN_OPTS="-Xms2g -Xmx2g" \ && export JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64" \ && tar -xzvf /apache-atlas-3.0.0-SNAPSHOT-server.tar.gz -C /opt \ @@ -57,22 +56,12 @@ COPY atlas-hub/repair_index.py /opt/apache-atlas/bin/ RUN chmod +x /opt/apache-atlas/bin/repair_index.py COPY atlas-hub/atlas_start.py.patch atlas-hub/atlas_config.py.patch /opt/apache-atlas/bin/ -COPY atlas-hub/pre-conf/ranger/lib/ /opt/apache-atlas/libext/ -COPY atlas-hub/pre-conf/ranger/install/conf.templates/enable/ /opt/apache-atlas/conf/ COPY atlas-hub/pre-conf/atlas-log4j.xml /opt/apache-atlas/conf/ COPY atlas-hub/pre-conf/atlas-log4j2.xml /opt/apache-atlas/conf/ -COPY atlas-hub/pre-conf/ranger/ /opt/ranger-atlas-plugin/ -COPY atlas-hub/env_change.sh / +COPY atlas-hub/pre-conf/atlas-auth/ /opt/apache-atlas/conf/ RUN curl https://repo1.maven.org/maven2/org/jolokia/jolokia-jvm/1.6.2/jolokia-jvm-1.6.2-agent.jar -o /opt/apache-atlas/libext/jolokia-jvm-agent.jar -RUN cd /opt/apache-atlas/bin \ - && sed "s~ATLAS_INSTALL_DIR~/opt/apache-atlas~g" /opt/ranger-atlas-plugin/install.properties > /tmp/install.properties \ - && cp /tmp/install.properties /opt/ranger-atlas-plugin/install.properties \ - && chmod +x /env_change.sh -# && patch -b -f < atlas_start.py.patch \ -# && patch -b -f < atlas_config.py.patch \ - RUN cd /opt/apache-atlas/bin \ && ./atlas_start.py -setup || true diff --git a/addons/hbase-bridge/pom.xml b/addons/hbase-bridge/pom.xml index a6ed51421f..fe5a711816 100644 --- a/addons/hbase-bridge/pom.xml +++ b/addons/hbase-bridge/pom.xml @@ -214,12 +214,6 @@ - - org.apache.hadoop - hadoop-auth - ${hadoop.version} - compile - org.apache.atlas atlas-client-v2 diff --git a/addons/kafka-bridge/pom.xml b/addons/kafka-bridge/pom.xml index 30fb53d27c..820b478441 100644 --- a/addons/kafka-bridge/pom.xml +++ b/addons/kafka-bridge/pom.xml @@ -131,12 +131,6 @@ - - org.apache.hadoop - hadoop-auth - ${hadoop.version} - compile - diff --git a/atlas-hub/env_change.sh b/atlas-hub/env_change.sh deleted file mode 100644 index 66e0718bbf..0000000000 --- a/atlas-hub/env_change.sh +++ /dev/null @@ -1,24 +0,0 @@ -#! /bin/bash - -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -sed -i -e "s~RANGER_SERVICE_URL~${RANGER_SERVICE_URL}~g" /opt/ranger-atlas-plugin/install.properties -sed -i -e "s~ATLAS_REPOSITORY_NAME~${ATLAS_REPOSITORY_NAME}~g" /opt/ranger-atlas-plugin/install.properties -bash /opt/ranger-atlas-plugin/enable-atlas-plugin.sh -sleep 10 \ No newline at end of file diff --git a/atlas-hub/pre-conf/ranger/install/conf.templates/enable/atlas-atlas-audit.xml b/atlas-hub/pre-conf/atlas-auth/atlas-atlas-audit.xml similarity index 100% rename from atlas-hub/pre-conf/ranger/install/conf.templates/enable/atlas-atlas-audit.xml rename to atlas-hub/pre-conf/atlas-auth/atlas-atlas-audit.xml diff --git a/atlas-hub/pre-conf/ranger/install/conf.templates/enable/atlas-atlas-security-changes.cfg b/atlas-hub/pre-conf/atlas-auth/atlas-atlas-security-changes.cfg similarity index 100% rename from atlas-hub/pre-conf/ranger/install/conf.templates/enable/atlas-atlas-security-changes.cfg rename to atlas-hub/pre-conf/atlas-auth/atlas-atlas-security-changes.cfg diff --git a/atlas-hub/pre-conf/ranger/install/conf.templates/enable/atlas-atlas-security.xml b/atlas-hub/pre-conf/atlas-auth/atlas-atlas-security.xml similarity index 100% rename from atlas-hub/pre-conf/ranger/install/conf.templates/enable/atlas-atlas-security.xml rename to atlas-hub/pre-conf/atlas-auth/atlas-atlas-security.xml diff --git a/atlas-hub/pre-conf/ranger/disable-atlas-plugin.sh b/atlas-hub/pre-conf/ranger/disable-atlas-plugin.sh deleted file mode 100755 index a15728f695..0000000000 --- a/atlas-hub/pre-conf/ranger/disable-atlas-plugin.sh +++ /dev/null @@ -1,795 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -function getInstallProperty() { - local propertyName=$1 - local propertyValue="" - - for file in "${COMPONENT_INSTALL_ARGS}" "${INSTALL_ARGS}" - do - if [ -f "${file}" ] - then - propertyValue=`grep "^${propertyName}[ \t]*=" ${file} | awk -F= '{ sub("^[ \t]*", "", $2); sub("[ \t]*$", "", $2); print $2 }'` - if [ "${propertyValue}" != "" ] - then - break - fi - fi - done - - echo ${propertyValue} -} - -# -# Base env variable for Ranger related files/directories -# -PROJ_NAME=ranger - -# -# The script should be run by "root" user -# - -if [ ! -w /etc/passwd ] -then - echo "ERROR: $0 script should be run as root." - exit 1 -fi - -#Check for JAVA_HOME -if [ "${JAVA_HOME}" == "" ] -then - echo "ERROR: JAVA_HOME environment property not defined, aborting installation." - exit 1 -fi - -# -# Identify the component, action from the script file -# - -basedir=`dirname $0` -if [ "${basedir}" = "." ] -then - basedir=`pwd` -elif [ "${basedir}" = ".." ] -then - basedir=`(cd .. ;pwd)` -fi - -# -# As this script is common to all component, find the component name based on the script-name -# - -COMPONENT_NAME=`basename $0 | cut -d. -f1 | sed -e 's:^disable-::' | sed -e 's:^enable-::'` - -echo "${COMPONENT_NAME}" | grep 'plugin' > /dev/null 2>&1 - -if [ $? -ne 0 ] -then - echo "$0 : is not applicable for component [${COMPONENT_NAME}]. It is applicable only for ranger plugin component; Exiting ..." - exit 0 -fi - -HCOMPONENT_NAME=`echo ${COMPONENT_NAME} | sed -e 's:-plugin::'` - -CFG_OWNER_INF="${HCOMPONENT_NAME}:${HCOMPONENT_NAME}" - -if [ "${HCOMPONENT_NAME}" = "hdfs" ] -then - HCOMPONENT_NAME="hadoop" -fi - -# -# Based on script name, identify if the action is enabled or disabled -# - -basename $0 | cut -d. -f1 | grep '^enable-' > /dev/null 2>&1 - -if [ $? -eq 0 ] -then - action=enable -else - action=disable -fi - - -# -# environment variables for enable|disable scripts -# - -PROJ_INSTALL_DIR=`(cd ${basedir} ; pwd)` -SET_ENV_SCRIPT_NAME=set-${COMPONENT_NAME}-env.sh -SET_ENV_SCRIPT_TEMPLATE=${PROJ_INSTALL_DIR}/install/conf.templates/enable/${SET_ENV_SCRIPT_NAME} -DEFAULT_XML_CONFIG=${PROJ_INSTALL_DIR}/install/conf.templates/default/configuration.xml -PROJ_LIB_DIR=${PROJ_INSTALL_DIR}/lib -PROJ_INSTALL_LIB_DIR="${PROJ_INSTALL_DIR}/install/lib" -INSTALL_ARGS="${PROJ_INSTALL_DIR}/install.properties" -COMPONENT_INSTALL_ARGS="${PROJ_INSTALL_DIR}/${COMPONENT_NAME}-install.properties" -JAVA=$JAVA_HOME/bin/java - -PLUGIN_DEPENDENT_LIB_DIR=lib/"${PROJ_NAME}-${COMPONENT_NAME}-impl" -PROJ_LIB_PLUGIN_DIR=${PROJ_INSTALL_DIR}/${PLUGIN_DEPENDENT_LIB_DIR} - -HCOMPONENT_INSTALL_DIR_NAME=$(getInstallProperty 'COMPONENT_INSTALL_DIR_NAME') - - -CUSTOM_USER=$(getInstallProperty 'CUSTOM_USER') -CUSTOM_USER=${CUSTOM_USER// } - -CUSTOM_GROUP=$(getInstallProperty 'CUSTOM_GROUP') -CUSTOM_GROUP=${CUSTOM_GROUP// } - -CUSTOM_GROUP_STATUS=${CUSTOM_GROUP}; -CUSTOM_USER_STATUS=${CUSTOM_USER}; -egrep "^$CUSTOM_GROUP" /etc/group >& /dev/null -if [ $? -ne 0 ] -then - CUSTOM_GROUP_STATUS="" -fi -id -u ${CUSTOM_USER} > /dev/null 2>&1 -if [ $? -ne 0 ] -then - CUSTOM_USER_STATUS="" -fi - -if [ ! -z "${CUSTOM_USER_STATUS}" ] && [ ! -z "${CUSTOM_GROUP_STATUS}" ] -then - echo "Custom user and group is available, using custom user and group." - CFG_OWNER_INF="${CUSTOM_USER}:${CUSTOM_GROUP}" -elif [ ! -z "${CUSTOM_USER_STATUS}" ] && [ -z "${CUSTOM_GROUP_STATUS}" ] -then - echo "Custom user is available, using custom user and default group." - CFG_OWNER_INF="${CUSTOM_USER}:${HCOMPONENT_NAME}" -elif [ -z "${CUSTOM_USER_STATUS}" ] && [ ! -z "${CUSTOM_GROUP_STATUS}" ] -then - echo "Custom group is available, using default user and custom group." - CFG_OWNER_INF="${HCOMPONENT_NAME}:${CUSTOM_GROUP}" -else - echo "Custom user and group are not available, using default user and group." - CFG_OWNER_INF="${HCOMPONENT_NAME}:${HCOMPONENT_NAME}" -fi - -if [ "${HCOMPONENT_INSTALL_DIR_NAME}" = "" ] -then - if [ "${HCOMPONENT_NAME}" = "knox" ]; - then - HCOMPONENT_INSTALL_DIR_NAME=$(getInstallProperty 'KNOX_HOME') - fi - if [ "${HCOMPONENT_INSTALL_DIR_NAME}" = "" ] - then - HCOMPONENT_INSTALL_DIR_NAME=${HCOMPONENT_NAME} - fi -fi - -firstletter=${HCOMPONENT_INSTALL_DIR_NAME:0:1} -if [ "$firstletter" = "/" ]; then - hdir=${HCOMPONENT_INSTALL_DIR_NAME} -else - hdir=${PROJ_INSTALL_DIR}/../${HCOMPONENT_INSTALL_DIR_NAME} -fi - -# -# TEST - START -# -if [ ! -d ${hdir} ] -then - mkdir -p ${hdir} -fi -# -# TEST - END -# -HCOMPONENT_INSTALL_DIR=`(cd ${hdir} ; pwd)` -HCOMPONENT_LIB_DIR=${HCOMPONENT_INSTALL_DIR}/lib -if [ "${HCOMPONENT_NAME}" = "knox" ]; then - HCOMPONENT_LIB_DIR=${HCOMPONENT_INSTALL_DIR}/ext -elif [ "${HCOMPONENT_NAME}" = "solr" ]; then - HCOMPONENT_LIB_DIR=${HCOMPONENT_INSTALL_DIR}/solr-webapp/webapp/WEB-INF/lib -elif [ "${HCOMPONENT_NAME}" = "kafka" ]; then - HCOMPONENT_LIB_DIR=${HCOMPONENT_INSTALL_DIR}/libs -elif [ "${HCOMPONENT_NAME}" = "storm" ]; then - HCOMPONENT_LIB_DIR=${HCOMPONENT_INSTALL_DIR}/extlib-daemon -elif [ "${HCOMPONENT_NAME}" = "atlas" ]; then - HCOMPONENT_LIB_DIR=${HCOMPONENT_INSTALL_DIR}/libext -elif [ "${HCOMPONENT_NAME}" = "hadoop" ] || - [ "${HCOMPONENT_NAME}" = "yarn" ]; then - HCOMPONENT_LIB_DIR=${HCOMPONENT_INSTALL_DIR}/share/hadoop/hdfs/lib -elif [ "${HCOMPONENT_NAME}" = "sqoop" ]; then - HCOMPONENT_LIB_DIR=${HCOMPONENT_INSTALL_DIR}/server/lib -elif [ "${HCOMPONENT_NAME}" = "kylin" ]; then - HCOMPONENT_LIB_DIR=${HCOMPONENT_INSTALL_DIR}/tomcat/webapps/kylin/WEB-INF/lib -elif [ "${HCOMPONENT_NAME}" = "elasticsearch" ]; then - HCOMPONENT_LIB_DIR=${HCOMPONENT_INSTALL_DIR}/plugins -elif [ "${HCOMPONENT_NAME}" = "presto" ]; then - HCOMPONENT_LIB_DIR=${HCOMPONENT_INSTALL_DIR}/plugin/ranger - if [ ! -d "${HCOMPONENT_LIB_DIR}" ]; then - echo "INFO: Creating ${HCOMPONENT_LIB_DIR}" - mkdir -p ${HCOMPONENT_LIB_DIR} - fi -fi - -HCOMPONENT_CONF_DIR=${HCOMPONENT_INSTALL_DIR}/conf -if [ "${HCOMPONENT_NAME}" = "solr" ]; then - HCOMPONENT_CONF_DIR=${HCOMPONENT_INSTALL_DIR}/resources - if [ ! -d $HCOMPONENT_CONF_DIR ]; then - install_owner=`ls -ld | cut -f 3 -d " "` - echo "INFO: Creating $HCOMPONENT_CONF_DIR" - mkdir -p $HCOMPONENT_CONF_DIR - echo "INFO: Changing ownership of $HCOMPONENT_CONF_DIR to $install_owner" - chown $install_owner:$install_owner $HCOMPONENT_CONF_DIR - fi -elif [ "${HCOMPONENT_NAME}" = "kafka" ]; then - HCOMPONENT_CONF_DIR=${HCOMPONENT_INSTALL_DIR}/config -elif [ "${HCOMPONENT_NAME}" = "hadoop" ]; then - HCOMPONENT_CONF_DIR=${HCOMPONENT_INSTALL_DIR}/etc/hadoop -elif [ "${HCOMPONENT_NAME}" = "yarn" ]; then - HCOMPONENT_CONF_DIR=${HCOMPONENT_INSTALL_DIR}/etc/hadoop -elif [ "${HCOMPONENT_NAME}" = "sqoop" ]; then - HCOMPONENT_CONF_DIR=${HCOMPONENT_INSTALL_DIR}/conf -elif [ "${HCOMPONENT_NAME}" = "elasticsearch" ]; then - HCOMPONENT_CONF_DIR=${HCOMPONENT_INSTALL_DIR}/config/ranger-elasticsearch-plugin - if [ ! -d $HCOMPONENT_CONF_DIR ]; then - echo "INFO: Creating $HCOMPONENT_CONF_DIR" - mkdir -p $HCOMPONENT_CONF_DIR - echo "INFO: Changing ownership of $HCOMPONENT_CONF_DIR to $CFG_OWNER_INF" - chown $CFG_OWNER_INF $HCOMPONENT_CONF_DIR - fi -elif [ "${HCOMPONENT_NAME}" = "presto" ]; then - HCOMPONENT_CONF_DIR=${HCOMPONENT_INSTALL_DIR}/etc -fi - -HCOMPONENT_ARCHIVE_CONF_DIR=${HCOMPONENT_CONF_DIR}/.archive -SET_ENV_SCRIPT=${HCOMPONENT_CONF_DIR}/${SET_ENV_SCRIPT_NAME} - - -if [ ! -d "${HCOMPONENT_INSTALL_DIR}" ] -then - echo "ERROR: Unable to find the install directory of component [${HCOMPONENT_NAME}]; dir [${HCOMPONENT_INSTALL_DIR}] not found." - echo "Exiting installation." - exit 1 -fi - -if [ ! -d "${HCOMPONENT_CONF_DIR}" ] -then - echo "ERROR: Unable to find the conf directory of component [${HCOMPONENT_NAME}]; dir [${HCOMPONENT_CONF_DIR}] not found." - echo "Exiting installation." - exit 1 -fi - -if [ ! -d "${HCOMPONENT_LIB_DIR}" ] -then - mkdir -p "${HCOMPONENT_LIB_DIR}" - if [ ! -d "${HCOMPONENT_LIB_DIR}" ] - then - echo "ERROR: Unable to find the lib directory of component [${HCOMPONENT_NAME}]; dir [${HCOMPONENT_LIB_DIR}] not found." - echo "Exiting installation." - exit 1 - fi -fi - -ambari_hive_install="N" -if [ "${HCOMPONENT_NAME}" = "hive" ] -then - HCOMPONENT_CONF_SERVER_DIR="${HCOMPONENT_CONF_DIR}"/conf.server - if [ -d "${HCOMPONENT_CONF_SERVER_DIR}" ] - then - ambari_hive_install="Y" - fi -fi - -# -# Common functions used by all enable/disable scripts -# - -log() { - echo "+ `date` : $*" -} - - -create_jceks() { - - alias=$1 - pass=$2 - jceksFile=$3 - - if [ -f "${jceksFile}" ] - then - jcebdir=`dirname ${jceksFile}` - jcebname=`basename ${jceksFile}` - archive_jce=${jcebdir}/.${jcebname}.`date '+%Y%m%d%H%M%S'` - log "Saving current JCE file: ${jceksFile} to ${archive_jce} ..." - cp ${jceksFile} ${archive_jce} - fi - - tempFile=/tmp/jce.$$.out - - $JAVA_HOME/bin/java -cp ":${PROJ_INSTALL_LIB_DIR}/*:" org.apache.ranger.credentialapi.buildks create "${alias}" -value "${pass}" -provider "jceks://file${jceksFile}" > ${tempFile} 2>&1 - - if [ $? -ne 0 ] - then - echo "Unable to store password in non-plain text format. Error: [`cat ${tempFile}`]" - echo "Exiting plugin installation" - rm -f ${tempFile} - exit 0 - fi - - rm -f ${tempFile} -} - -log "${HCOMPONENT_NAME}: lib folder=$HCOMPONENT_LIB_DIR conf folder=$HCOMPONENT_CONF_DIR" - -# -# If there is a set-ranger-${COMPONENT}-env.sh, install it -# -dt=`date '+%Y%m%d-%H%M%S'` - -if [ -f "${SET_ENV_SCRIPT_TEMPLATE}" ] -then - # - # If the setenv script already exists, move it to the archive folder - # - if [ -f "${SET_ENV_SCRIPT}" ] - then - if [ ! -d "${HCOMPONENT_ARCHIVE_CONF_DIR}" ] - then - mkdir -p ${HCOMPONENT_ARCHIVE_CONF_DIR} - fi - log "Saving current ${SET_ENV_SCRIPT_NAME} to ${HCOMPONENT_ARCHIVE_CONF_DIR} ..." - mv ${SET_ENV_SCRIPT} ${HCOMPONENT_ARCHIVE_CONF_DIR}/${SET_ENV_SCRIPT_NAME}.${dt} - fi - - if [ "${action}" = "enable" ] - then - - cp ${SET_ENV_SCRIPT_TEMPLATE} ${SET_ENV_SCRIPT} - - DEST_SCRIPT_FILE=${HCOMPONENT_INSTALL_DIR}/libexec/${HCOMPONENT_NAME}-config.sh - - DEST_SCRIPT_ARCHIVE_FILE=${HCOMPONENT_INSTALL_DIR}/libexec/.${HCOMPONENT_NAME}-config.sh.${dt} - - if [ -f "${DEST_SCRIPT_FILE}" ] - then - - log "Saving current ${DEST_SCRIPT_FILE} to ${DEST_SCRIPT_ARCHIVE_FILE} ..." - - cp ${DEST_SCRIPT_FILE} ${DEST_SCRIPT_ARCHIVE_FILE} - - grep 'xasecure-.*-env.sh' ${DEST_SCRIPT_FILE} > /dev/null 2>&1 - if [ $? -eq 0 ] - then - ts=`date '+%Y%m%d%H%M%S'` - grep -v 'xasecure-.*-env.sh' ${DEST_SCRIPT_FILE} > ${DEST_SCRIPT_FILE}.${ts} - if [ $? -eq 0 ] - then - log "Removing old reference to xasecure setenv source ..." - cat ${DEST_SCRIPT_FILE}.${ts} > ${DEST_SCRIPT_FILE} - rm -f ${DEST_SCRIPT_FILE}.${ts} - fi - fi - - grep "[ \t]*.[ \t]*${SET_ENV_SCRIPT}" ${DEST_SCRIPT_FILE} > /dev/null - if [ $? -ne 0 ] - then - log "Appending sourcing script, ${SET_ENV_SCRIPT_NAME} in the file: ${DEST_SCRIPT_FILE} " - cat >> ${DEST_SCRIPT_FILE} <\n`date`\n" > ${HCOMPONENT_CONF_DIR}/ranger-security.xml - chown ${CFG_OWNER_INF} ${HCOMPONENT_CONF_DIR}/ranger-security.xml - chmod a+r ${HCOMPONENT_CONF_DIR}/ranger-security.xml - for cf in ${PROJ_INSTALL_DIR}/install/conf.templates/${action}/*.xml - do - cfb=`basename ${cf}` - if [ -f "${HCOMPONENT_CONF_DIR}/${cfb}" ] - then - log "Saving ${HCOMPONENT_CONF_DIR}/${cfb} to ${HCOMPONENT_CONF_DIR}/.${cfb}.${dt} ..." - cp ${HCOMPONENT_CONF_DIR}/${cfb} ${HCOMPONENT_CONF_DIR}/.${cfb}.${dt} - fi - cp ${cf} ${HCOMPONENT_CONF_DIR}/ - chown ${CFG_OWNER_INF} ${HCOMPONENT_CONF_DIR}/${cfb} - chmod a+r ${HCOMPONENT_CONF_DIR}/${cfb} - done - else - if [ -f ${HCOMPONENT_CONF_DIR}/ranger-security.xml ] - then - mv ${HCOMPONENT_CONF_DIR}/ranger-security.xml ${HCOMPONENT_CONF_DIR}/.ranger-security.xml.`date '+%Y%m%d%H%M%S'` - fi - fi - - # - # Ensure that POLICY_CACHE_FILE_PATH is accessible - # - REPO_NAME=$(getInstallProperty 'REPOSITORY_NAME') - export POLICY_CACHE_FILE_PATH=/etc/${PROJ_NAME}/${REPO_NAME}/policycache - export CREDENTIAL_PROVIDER_FILE=/etc/${PROJ_NAME}/${REPO_NAME}/cred.jceks - if [ ! -d ${POLICY_CACHE_FILE_PATH} ] - then - mkdir -p ${POLICY_CACHE_FILE_PATH} - fi - chmod a+rx /etc/${PROJ_NAME} - chmod a+rx /etc/${PROJ_NAME}/${REPO_NAME} - chmod a+rx ${POLICY_CACHE_FILE_PATH} - chown -R ${CFG_OWNER_INF} /etc/${PROJ_NAME}/${REPO_NAME} - - for f in ${PROJ_INSTALL_DIR}/install/conf.templates/${action}/*.cfg - do - if [ -f "${f}" ] - then - fn=`basename $f` - orgfn=`echo $fn | sed -e 's:-changes.cfg:.xml:'` - fullpathorgfn="${HCOMPONENT_CONF_DIR}/${orgfn}" - if [ ! -f ${fullpathorgfn} ] - then - if [ -f ${DEFAULT_XML_CONFIG} ] - then - log "Creating default file from [${DEFAULT_XML_CONFIG}] for [${fullpathorgfn}] .." - cp ${DEFAULT_XML_CONFIG} ${fullpathorgfn} - chown ${CFG_OWNER_INF} ${fullpathorgfn} - chmod a+r ${fullpathorgfn} - else - echo "ERROR: Unable to find ${fullpathorgfn}" - exit 1 - fi - fi - archivefn="${HCOMPONENT_CONF_DIR}/.${orgfn}.${dt}" - newfn="${HCOMPONENT_CONF_DIR}/.${orgfn}-new.${dt}" - log "Saving current config file: ${fullpathorgfn} to ${archivefn} ..." - cp ${fullpathorgfn} ${archivefn} - if [ $? -eq 0 ] - then - ${JAVA} -cp "${INSTALL_CP}" org.apache.ranger.utils.install.XmlConfigChanger -i ${archivefn} -o ${newfn} -c ${f} -p ${INSTALL_ARGS} - if [ $? -eq 0 ] - then - diff -w ${newfn} ${fullpathorgfn} > /dev/null 2>&1 - if [ $? -ne 0 ] - then - cat ${newfn} > ${fullpathorgfn} - fi - - # For Ambari install copy the .xml to conf.server also - if [ "${ambari_hive_install}" = "Y" ] - then - fullpathorgHS2fn="${HCOMPONENT_CONF_SERVER_DIR}/${orgfn}" - archiveHS2fn="${HCOMPONENT_CONF_SERVER_DIR}/.${orgfn}.${dt}" - newHS2fn="${HCOMPONENT_CONF_SERVER_DIR}/.${orgfn}-new.${dt}" - log "Saving current conf.server file: ${fullpathorgHS2fn} to ${archiveHS2fn} ..." - if [ -f ${fullpathorgHS2fn} ] - then - cp ${fullpathorgHS2fn} ${archiveHS2fn} - fi - cp ${fullpathorgfn} ${HCOMPONENT_CONF_SERVER_DIR}/${orgfn} - chown ${CFG_OWNER_INF} ${HCOMPONENT_CONF_SERVER_DIR}/${orgfn} - fi - - else - echo "ERROR: Unable to make changes to config. file: ${fullpathorgfn}" - echo "exiting ...." - exit 1 - fi - else - echo "ERROR: Unable to save config. file: ${fullpathorgfn} to ${archivefn}" - echo "exiting ...." - exit 1 - fi - fi - done - if [ "${HCOMPONENT_NAME}" = "hbase" ] || [ "${HCOMPONENT_NAME}" = "storm" ]; - then - chmod 644 ${HCOMPONENT_CONF_DIR}/* - fi -fi - -# -# Create library link -# -if [ "${action}" = "enable" ] -then - dt=`date '+%Y%m%d%H%M%S'` - for f in ${PROJ_LIB_DIR}/* - do - if [ -f "${f}" ] || [ -d "${f}" ] - then - bn=`basename $f` - if [ -f ${HCOMPONENT_LIB_DIR}/${bn} ] || [ -d ${HCOMPONENT_LIB_DIR}/${bn} ] - then - log "Saving lib file: ${HCOMPONENT_LIB_DIR}/${bn} to ${HCOMPONENT_LIB_DIR}/.${bn}.${dt} ..." - mv ${HCOMPONENT_LIB_DIR}/${bn} ${HCOMPONENT_LIB_DIR}/.${bn}.${dt} - fi - if [ ! -f ${HCOMPONENT_LIB_DIR}/${bn} ] && [ ! -d ${HCOMPONENT_LIB_DIR}/${bn} ] - then - ln -s ${f} ${HCOMPONENT_LIB_DIR}/${bn} - fi - fi - done - - # - # Encrypt the password and keep it secure in Credential Provider API - # - CredFile=${CREDENTIAL_PROVIDER_FILE} - if ! [ `echo ${CredFile} | grep '^/.*'` ] - then - echo "ERROR:Please enter the Credential File Store with proper file path" - exit 1 - fi - - pardir=`dirname ${CredFile}` - - if [ ! -d "${pardir}" ] - then - mkdir -p "${pardir}" - if [ $? -ne 0 ] - then - echo "ERROR: Unable to create credential store file path" - exit 1 - fi - chmod a+rx "${pardir}" - fi - - # - # Generate Credential Provider file and Credential for SSL KEYSTORE AND TRUSTSTORE - # - sslkeystoreAlias="sslKeyStore" - sslkeystoreCred=$(getInstallProperty 'SSL_KEYSTORE_PASSWORD') - create_jceks "${sslkeystoreAlias}" "${sslkeystoreCred}" "${CredFile}" - ssltruststoreAlias="sslTrustStore" - ssltruststoreCred=$(getInstallProperty 'SSL_TRUSTSTORE_PASSWORD') - create_jceks "${ssltruststoreAlias}" "${ssltruststoreCred}" "${CredFile}" - chown ${CFG_OWNER_INF} ${CredFile} - # - # To allow all users in the server (where Hive CLI and HBase CLI is used), - # user needs to have read access for the credential file. - # - chmod a+r ${CredFile} -fi - -# -# Knox specific configuration -# -# - -if [ "${HCOMPONENT_NAME}" = "knox" ] -then - if [ "${action}" = "enable" ] - then - authFrom="AclsAuthz" - authTo="XASecurePDPKnox" - else - authTo="AclsAuthz" - authFrom="XASecurePDPKnox" - fi - - dt=`date '+%Y%m%d%H%M%S'` - for fn in `ls ${HCOMPONENT_CONF_DIR}/topologies/*.xml 2> /dev/null` - do - if [ -f "${fn}" ] - then - dn=`dirname ${fn}` - bn=`basename ${fn}` - bf=${dn}/.${bn}.${dt} - echo "backup of ${fn} to ${bf} ..." - cp ${fn} ${bf} - echo "Updating topology file: [${fn}] ... " - cat ${fn} | sed -e "s-${authFrom}-${authTo}-" > ${fn}.${dt}.new - if [ $? -eq 0 ] - then - cat ${fn}.${dt}.new > ${fn} - rm ${fn}.${dt}.new - fi - fi - done -fi - -if [ "${HCOMPONENT_NAME}" = "storm" ] -then - CFG_FILE=${HCOMPONENT_CONF_DIR}/storm.yaml - ARCHIVE_FILE=${HCOMPONENT_CONF_DIR}/.storm.yaml.`date '+%Y%m%d%H%M%S'` - - if [ -f "${CFG_FILE}" ] - then - cp ${CFG_FILE} ${ARCHIVE_FILE} - - if [ "${action}" = "enable" ] - then - awk -F: 'BEGIN { - configured = 0 ; - } - { - if ($1 == "nimbus.authorizer") { - if ($2 ~ /^[ \t]*"org.apache.ranger.authorization.storm.authorizer.RangerStormAuthorizer"[ \t]*$/) { - configured = 1 ; - printf("%s\n",$0) ; - } - else { - printf("#%s\n",$0); - printf("nimbus.authorizer: \"org.apache.ranger.authorization.storm.authorizer.RangerStormAuthorizer\"\n") ; - configured = 1 ; - } - } - else { - printf("%s\n",$0) ; - } - } - END { - if (configured == 0) { - printf("nimbus.authorizer: \"org.apache.ranger.authorization.storm.authorizer.RangerStormAuthorizer\"\n") ; - } - }' ${CFG_FILE} > ${CFG_FILE}.new && cat ${CFG_FILE}.new > ${CFG_FILE} && rm -f ${CFG_FILE}.new - - else - awk -F: 'BEGIN { - configured = 0 ; - } - { - if ($1 == "nimbus.authorizer") { - if ($2 ~ /^[ \t]*"org.apache.storm.security.auth.authorizer.SimpleACLAuthorizer"[ \t]*$/) { - configured = 1 ; - printf("%s\n",$0) ; - } - else { - printf("#%s\n",$0); - printf("nimbus.authorizer: \"org.apache.storm.security.auth.authorizer.SimpleACLAuthorizer\"\n") ; - configured = 1 ; - } - } - else { - printf("%s\n",$0) ; - } - } - END { - if (configured == 0) { - printf("nimbus.authorizer: \"org.apache.storm.security.auth.authorizer.SimpleACLAuthorizer\"\n") ; - } - }' ${CFG_FILE} > ${CFG_FILE}.new && cat ${CFG_FILE}.new > ${CFG_FILE} && rm -f ${CFG_FILE}.new - fi - fi -fi - -#Check Properties whether in File, return code 1 if not exist -#$1 -> propertyName; $2 -> fileName -checkPropertyInFile(){ - validate=$(sed '/^\#/d' $2 | grep "^$1" | tail -n 1 | cut -d "=" -f1-) # for validation - if test -z "$validate" ; then return 1; fi -} - -#Add Properties to File -#$1 -> propertyName; $2 -> newPropertyValue; $3 -> fileName -addPropertyToFile(){ - echo "$1=$2">>$3 - validate=$(sed '/^\#/d' $3 | grep "^$1" | tail -n 1 | cut -d "=" -f2-) # for validation - if test -z "$validate" ; then log "[E] Failed to add properties '$1' to $3 file!"; exit 1; fi - echo "Property $1 added successfully with : '$2'" -} - -#Update Properties to File -#$1 -> propertyName; $2 -> newPropertyValue; $3 -> fileName -updatePropertyToFile(){ - sed -i 's@^'$1'=[^ ]*$@'$1'='$2'@g' $3 - validate=$(sed '/^\#/d' $3 | grep "^$1" | tail -n 1 | cut -d "=" -f2-) # for validation - if test -z "$validate" ; then log "[E] '$1' not found in $3 file while Updating....!!"; exit 1; fi - echo "Property $1 updated successfully with : '$2'" -} - -#Add or Update Properties to File -#$1 -> propertyName; $2 -> newPropertyValue; $3 -> fileName -addOrUpdatePropertyToFile(){ - checkPropertyInFile $1 $3 - if [ $? -eq 1 ] - then - addPropertyToFile $1 $2 $3 - else - updatePropertyToFile $1 $2 $3 - fi -} - -if [ "${HCOMPONENT_NAME}" = "sqoop" ] -then - if [ "${action}" = "enable" ] - then - authName="org.apache.ranger.authorization.sqoop.authorizer.RangerSqoopAuthorizer" - else - authName="" - fi - - dt=`date '+%Y%m%d%H%M%S'` - fn=`ls ${HCOMPONENT_CONF_DIR}/sqoop.properties 2> /dev/null` - if [ -f "${fn}" ] - then - dn=`dirname ${fn}` - bn=`basename ${fn}` - bf=${dn}/.${bn}.${dt} - echo "backup of ${fn} to ${bf} ..." - cp ${fn} ${bf} - echo "Add or Update properties file: [${fn}] ... " - addOrUpdatePropertyToFile org.apache.sqoop.security.authorization.validator $authName ${fn} - fi -fi - -if [ "${HCOMPONENT_NAME}" = "kylin" ] -then - if [ "${action}" = "enable" ] - then - authName="org.apache.ranger.authorization.kylin.authorizer.RangerKylinAuthorizer" - else - authName="" - fi - - dt=`date '+%Y%m%d%H%M%S'` - fn=`ls ${HCOMPONENT_CONF_DIR}/kylin.properties 2> /dev/null` - if [ -f "${fn}" ] - then - dn=`dirname ${fn}` - bn=`basename ${fn}` - bf=${dn}/.${bn}.${dt} - echo "backup of ${fn} to ${bf} ..." - cp ${fn} ${bf} - echo "Add or Update properties file: [${fn}] ... " - addOrUpdatePropertyToFile kylin.server.external-acl-provider $authName ${fn} - fi -fi - -if [ "${HCOMPONENT_NAME}" = "presto" ] -then - if [ "${action}" = "enable" ] - then - controlName="ranger" - else - controlName="" - fi - dt=`date '+%Y%m%d%H%M%S'` - fn=`ls ${HCOMPONENT_CONF_DIR}/access-control.properties 2> /dev/null` - if [ -f "${fn}" ] - then - dn=`dirname ${fn}` - bn=`basename ${fn}` - bf=${dn}/.${bn}.${dt} - echo "backup of ${fn} to ${bf} ..." - cp ${fn} ${bf} - else - fn=${HCOMPONENT_CONF_DIR}/access-control.properties - fi - echo "Add or Update properties file: [${fn}] ... " - addOrUpdatePropertyToFile access-control.name $controlName ${fn} - echo "Linking config files" - cd ${HCOMPONENT_LIB_DIR}/ranger-presto-plugin-impl/ - ln -sf ${HCOMPONENT_CONF_DIR} conf -fi - - -# -# Set notice to restart the ${HCOMPONENT_NAME} -# - -echo "Ranger Plugin for ${HCOMPONENT_NAME} has been ${action}d. Please restart ${HCOMPONENT_NAME} to ensure that changes are effective." - -exit 0 diff --git a/atlas-hub/pre-conf/ranger/enable-atlas-plugin.sh b/atlas-hub/pre-conf/ranger/enable-atlas-plugin.sh deleted file mode 100755 index 10c7fcfb0a..0000000000 --- a/atlas-hub/pre-conf/ranger/enable-atlas-plugin.sh +++ /dev/null @@ -1,798 +0,0 @@ -#!/bin/bash -JAVA_HOME='/usr/lib/jvm/java-1.8.0-openjdk-amd64' - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -function getInstallProperty() { - local propertyName=$1 - local propertyValue="" - - for file in "${COMPONENT_INSTALL_ARGS}" "${INSTALL_ARGS}" - do - if [ -f "${file}" ] - then - propertyValue=`grep "^${propertyName}[ \t]*=" ${file} | awk -F= '{ sub("^[ \t]*", "", $2); sub("[ \t]*$", "", $2); print $2 }'` - if [ "${propertyValue}" != "" ] - then - break - fi - fi - done - - echo ${propertyValue} -} - -# -# Base env variable for Ranger related files/directories -# -PROJ_NAME=ranger - -# -# The script should be run by "root" user -# - -if [ ! -w /etc/passwd ] -then - echo "ERROR: $0 script should be run as root." - exit 1 -fi - -#Check for JAVA_HOME -if [ "${JAVA_HOME}" == "" ] -then - echo "ERROR: JAVA_HOME environment property not defined, aborting installation." - exit 1 -fi - -# -# Identify the component, action from the script file -# - -basedir=`dirname $0` -if [ "${basedir}" = "." ] -then - basedir=`pwd` -elif [ "${basedir}" = ".." ] -then - basedir=`(cd .. ;pwd)` -fi - -# -# As this script is common to all component, find the component name based on the script-name -# - -COMPONENT_NAME=`basename $0 | cut -d. -f1 | sed -e 's:^disable-::' | sed -e 's:^enable-::'` - -echo "${COMPONENT_NAME}" | grep 'plugin' > /dev/null 2>&1 - -if [ $? -ne 0 ] -then - echo "$0 : is not applicable for component [${COMPONENT_NAME}]. It is applicable only for ranger plugin component; Exiting ..." - exit 0 -fi - -HCOMPONENT_NAME=`echo ${COMPONENT_NAME} | sed -e 's:-plugin::'` - -CFG_OWNER_INF="${HCOMPONENT_NAME}:${HCOMPONENT_NAME}" - -if [ "${HCOMPONENT_NAME}" = "hdfs" ] -then - HCOMPONENT_NAME="hadoop" -fi - -# -# Based on script name, identify if the action is enabled or disabled -# - -basename $0 | cut -d. -f1 | grep '^enable-' > /dev/null 2>&1 - -if [ $? -eq 0 ] -then - action=enable -else - action=disable -fi - - -# -# environment variables for enable|disable scripts -# - -PROJ_INSTALL_DIR=`(cd ${basedir} ; pwd)` -SET_ENV_SCRIPT_NAME=set-${COMPONENT_NAME}-env.sh -SET_ENV_SCRIPT_TEMPLATE=${PROJ_INSTALL_DIR}/install/conf.templates/enable/${SET_ENV_SCRIPT_NAME} -DEFAULT_XML_CONFIG=${PROJ_INSTALL_DIR}/install/conf.templates/default/configuration.xml -PROJ_LIB_DIR=${PROJ_INSTALL_DIR}/lib -PROJ_INSTALL_LIB_DIR="${PROJ_INSTALL_DIR}/install/lib" -INSTALL_ARGS="${PROJ_INSTALL_DIR}/install.properties" -COMPONENT_INSTALL_ARGS="${PROJ_INSTALL_DIR}/${COMPONENT_NAME}-install.properties" -JAVA=$JAVA_HOME/bin/java - -PLUGIN_DEPENDENT_LIB_DIR=lib/"${PROJ_NAME}-${COMPONENT_NAME}-impl" -PROJ_LIB_PLUGIN_DIR=${PROJ_INSTALL_DIR}/${PLUGIN_DEPENDENT_LIB_DIR} - -HCOMPONENT_INSTALL_DIR_NAME=$(getInstallProperty 'COMPONENT_INSTALL_DIR_NAME') - - -CUSTOM_USER=$(getInstallProperty 'CUSTOM_USER') -CUSTOM_USER=${CUSTOM_USER// } - -CUSTOM_GROUP=$(getInstallProperty 'CUSTOM_GROUP') -CUSTOM_GROUP=${CUSTOM_GROUP// } - -CUSTOM_GROUP_STATUS=${CUSTOM_GROUP}; -CUSTOM_USER_STATUS=${CUSTOM_USER}; -egrep "^$CUSTOM_GROUP" /etc/group >& /dev/null -if [ $? -ne 0 ] -then - CUSTOM_GROUP_STATUS="" -fi -id -u ${CUSTOM_USER} > /dev/null 2>&1 -if [ $? -ne 0 ] -then - CUSTOM_USER_STATUS="" -fi - -if [ ! -z "${CUSTOM_USER_STATUS}" ] && [ ! -z "${CUSTOM_GROUP_STATUS}" ] -then - echo "Custom user and group is available, using custom user and group." - CFG_OWNER_INF="${CUSTOM_USER}:${CUSTOM_GROUP}" -elif [ ! -z "${CUSTOM_USER_STATUS}" ] && [ -z "${CUSTOM_GROUP_STATUS}" ] -then - echo "Custom user is available, using custom user and default group." - CFG_OWNER_INF="${CUSTOM_USER}:${HCOMPONENT_NAME}" -elif [ -z "${CUSTOM_USER_STATUS}" ] && [ ! -z "${CUSTOM_GROUP_STATUS}" ] -then - echo "Custom group is available, using default user and custom group." - CFG_OWNER_INF="${HCOMPONENT_NAME}:${CUSTOM_GROUP}" -else - echo "Custom user and group are not available, using default user and group." - CFG_OWNER_INF="${HCOMPONENT_NAME}:${HCOMPONENT_NAME}" -fi - -if [ "${HCOMPONENT_INSTALL_DIR_NAME}" = "" ] -then - if [ "${HCOMPONENT_NAME}" = "knox" ]; - then - HCOMPONENT_INSTALL_DIR_NAME=$(getInstallProperty 'KNOX_HOME') - fi - if [ "${HCOMPONENT_INSTALL_DIR_NAME}" = "" ] - then - HCOMPONENT_INSTALL_DIR_NAME=${HCOMPONENT_NAME} - fi -fi - -firstletter=${HCOMPONENT_INSTALL_DIR_NAME:0:1} -if [ "$firstletter" = "/" ]; then - hdir=${HCOMPONENT_INSTALL_DIR_NAME} -else - hdir=${PROJ_INSTALL_DIR}/../${HCOMPONENT_INSTALL_DIR_NAME} -fi - -# -# TEST - START -# -if [ ! -d ${hdir} ] -then - mkdir -p ${hdir} -fi -# -# TEST - END -# -HCOMPONENT_INSTALL_DIR=`(cd ${hdir} ; pwd)` -HCOMPONENT_LIB_DIR=${HCOMPONENT_INSTALL_DIR}/lib -if [ "${HCOMPONENT_NAME}" = "knox" ]; then - HCOMPONENT_LIB_DIR=${HCOMPONENT_INSTALL_DIR}/ext -elif [ "${HCOMPONENT_NAME}" = "solr" ]; then - HCOMPONENT_LIB_DIR=${HCOMPONENT_INSTALL_DIR}/solr-webapp/webapp/WEB-INF/lib -elif [ "${HCOMPONENT_NAME}" = "kafka" ]; then - HCOMPONENT_LIB_DIR=${HCOMPONENT_INSTALL_DIR}/libs -elif [ "${HCOMPONENT_NAME}" = "storm" ]; then - HCOMPONENT_LIB_DIR=${HCOMPONENT_INSTALL_DIR}/extlib-daemon -elif [ "${HCOMPONENT_NAME}" = "atlas" ]; then - HCOMPONENT_LIB_DIR=${HCOMPONENT_INSTALL_DIR}/libext -elif [ "${HCOMPONENT_NAME}" = "hadoop" ] || - [ "${HCOMPONENT_NAME}" = "yarn" ]; then - HCOMPONENT_LIB_DIR=${HCOMPONENT_INSTALL_DIR}/share/hadoop/hdfs/lib -elif [ "${HCOMPONENT_NAME}" = "sqoop" ]; then - HCOMPONENT_LIB_DIR=${HCOMPONENT_INSTALL_DIR}/server/lib -elif [ "${HCOMPONENT_NAME}" = "kylin" ]; then - HCOMPONENT_LIB_DIR=${HCOMPONENT_INSTALL_DIR}/tomcat/webapps/kylin/WEB-INF/lib -elif [ "${HCOMPONENT_NAME}" = "elasticsearch" ]; then - HCOMPONENT_LIB_DIR=${HCOMPONENT_INSTALL_DIR}/plugins -elif [ "${HCOMPONENT_NAME}" = "presto" ]; then - HCOMPONENT_LIB_DIR=${HCOMPONENT_INSTALL_DIR}/plugin/ranger - if [ ! -d "${HCOMPONENT_LIB_DIR}" ]; then - echo "INFO: Creating ${HCOMPONENT_LIB_DIR}" - mkdir -p ${HCOMPONENT_LIB_DIR} - fi -fi - -HCOMPONENT_CONF_DIR=${HCOMPONENT_INSTALL_DIR}/conf -if [ "${HCOMPONENT_NAME}" = "solr" ]; then - HCOMPONENT_CONF_DIR=${HCOMPONENT_INSTALL_DIR}/resources - if [ ! -d $HCOMPONENT_CONF_DIR ]; then - install_owner=`ls -ld | cut -f 3 -d " "` - echo "INFO: Creating $HCOMPONENT_CONF_DIR" - mkdir -p $HCOMPONENT_CONF_DIR - echo "INFO: Changing ownership of $HCOMPONENT_CONF_DIR to $install_owner" - chown $install_owner:$install_owner $HCOMPONENT_CONF_DIR - fi -elif [ "${HCOMPONENT_NAME}" = "kafka" ]; then - HCOMPONENT_CONF_DIR=${HCOMPONENT_INSTALL_DIR}/config -elif [ "${HCOMPONENT_NAME}" = "hadoop" ]; then - HCOMPONENT_CONF_DIR=${HCOMPONENT_INSTALL_DIR}/etc/hadoop -elif [ "${HCOMPONENT_NAME}" = "yarn" ]; then - HCOMPONENT_CONF_DIR=${HCOMPONENT_INSTALL_DIR}/etc/hadoop -elif [ "${HCOMPONENT_NAME}" = "sqoop" ]; then - HCOMPONENT_CONF_DIR=${HCOMPONENT_INSTALL_DIR}/conf -elif [ "${HCOMPONENT_NAME}" = "elasticsearch" ]; then - HCOMPONENT_CONF_DIR=${HCOMPONENT_INSTALL_DIR}/config/ranger-elasticsearch-plugin - if [ ! -d $HCOMPONENT_CONF_DIR ]; then - echo "INFO: Creating $HCOMPONENT_CONF_DIR" - mkdir -p $HCOMPONENT_CONF_DIR - echo "INFO: Changing ownership of $HCOMPONENT_CONF_DIR to $CFG_OWNER_INF" - chown $CFG_OWNER_INF $HCOMPONENT_CONF_DIR - fi -elif [ "${HCOMPONENT_NAME}" = "presto" ]; then - HCOMPONENT_CONF_DIR=${HCOMPONENT_INSTALL_DIR}/etc -fi - -HCOMPONENT_ARCHIVE_CONF_DIR=${HCOMPONENT_CONF_DIR}/.archive -SET_ENV_SCRIPT=${HCOMPONENT_CONF_DIR}/${SET_ENV_SCRIPT_NAME} - - -if [ ! -d "${HCOMPONENT_INSTALL_DIR}" ] -then - echo "ERROR: Unable to find the install directory of component [${HCOMPONENT_NAME}]; dir [${HCOMPONENT_INSTALL_DIR}] not found." - echo "Exiting installation." - exit 1 -fi - -if [ ! -d "${HCOMPONENT_CONF_DIR}" ] -then - echo "ERROR: Unable to find the conf directory of component [${HCOMPONENT_NAME}]; dir [${HCOMPONENT_CONF_DIR}] not found." - echo "Exiting installation." - exit 1 -fi - -if [ ! -d "${HCOMPONENT_LIB_DIR}" ] -then - mkdir -p "${HCOMPONENT_LIB_DIR}" - if [ ! -d "${HCOMPONENT_LIB_DIR}" ] - then - echo "ERROR: Unable to find the lib directory of component [${HCOMPONENT_NAME}]; dir [${HCOMPONENT_LIB_DIR}] not found." - echo "Exiting installation." - exit 1 - fi -fi - -ambari_hive_install="N" -if [ "${HCOMPONENT_NAME}" = "hive" ] -then - HCOMPONENT_CONF_SERVER_DIR="${HCOMPONENT_CONF_DIR}"/conf.server - if [ -d "${HCOMPONENT_CONF_SERVER_DIR}" ] - then - ambari_hive_install="Y" - fi -fi - -# -# Common functions used by all enable/disable scripts -# - -log() { - echo "+ `date` : $*" -} - - -create_jceks() { - - alias=$1 - pass=$2 - jceksFile=$3 - - if [ -f "${jceksFile}" ] - then - jcebdir=`dirname ${jceksFile}` - jcebname=`basename ${jceksFile}` - archive_jce=${jcebdir}/.${jcebname}.`date '+%Y%m%d%H%M%S'` - log "Saving current JCE file: ${jceksFile} to ${archive_jce} ..." - cp ${jceksFile} ${archive_jce} - fi - - tempFile=/tmp/jce.$$.out - - $JAVA_HOME/bin/java -cp ":${PROJ_INSTALL_LIB_DIR}/*:" org.apache.ranger.credentialapi.buildks create "${alias}" -value "${pass}" -provider "jceks://file${jceksFile}" > ${tempFile} 2>&1 - - if [ $? -ne 0 ] - then - echo "Unable to store password in non-plain text format. Error: [`cat ${tempFile}`]" - echo "Exiting plugin installation" - rm -f ${tempFile} - exit 0 - fi - - rm -f ${tempFile} -} - -log "${HCOMPONENT_NAME}: lib folder=$HCOMPONENT_LIB_DIR conf folder=$HCOMPONENT_CONF_DIR" - -# -# If there is a set-ranger-${COMPONENT}-env.sh, install it -# -dt=`date '+%Y%m%d-%H%M%S'` - -if [ -f "${SET_ENV_SCRIPT_TEMPLATE}" ] -then - # - # If the setenv script already exists, move it to the archive folder - # - if [ -f "${SET_ENV_SCRIPT}" ] - then - if [ ! -d "${HCOMPONENT_ARCHIVE_CONF_DIR}" ] - then - mkdir -p ${HCOMPONENT_ARCHIVE_CONF_DIR} - fi - log "Saving current ${SET_ENV_SCRIPT_NAME} to ${HCOMPONENT_ARCHIVE_CONF_DIR} ..." - mv ${SET_ENV_SCRIPT} ${HCOMPONENT_ARCHIVE_CONF_DIR}/${SET_ENV_SCRIPT_NAME}.${dt} - fi - - if [ "${action}" = "enable" ] - then - - cp ${SET_ENV_SCRIPT_TEMPLATE} ${SET_ENV_SCRIPT} - - DEST_SCRIPT_FILE=${HCOMPONENT_INSTALL_DIR}/libexec/${HCOMPONENT_NAME}-config.sh - - DEST_SCRIPT_ARCHIVE_FILE=${HCOMPONENT_INSTALL_DIR}/libexec/.${HCOMPONENT_NAME}-config.sh.${dt} - - if [ -f "${DEST_SCRIPT_FILE}" ] - then - - log "Saving current ${DEST_SCRIPT_FILE} to ${DEST_SCRIPT_ARCHIVE_FILE} ..." - - cp ${DEST_SCRIPT_FILE} ${DEST_SCRIPT_ARCHIVE_FILE} - - grep 'xasecure-.*-env.sh' ${DEST_SCRIPT_FILE} > /dev/null 2>&1 - if [ $? -eq 0 ] - then - ts=`date '+%Y%m%d%H%M%S'` - grep -v 'xasecure-.*-env.sh' ${DEST_SCRIPT_FILE} > ${DEST_SCRIPT_FILE}.${ts} - if [ $? -eq 0 ] - then - log "Removing old reference to xasecure setenv source ..." - cat ${DEST_SCRIPT_FILE}.${ts} > ${DEST_SCRIPT_FILE} - rm -f ${DEST_SCRIPT_FILE}.${ts} - fi - fi - - grep "[ \t]*.[ \t]*${SET_ENV_SCRIPT}" ${DEST_SCRIPT_FILE} > /dev/null - if [ $? -ne 0 ] - then - log "Appending sourcing script, ${SET_ENV_SCRIPT_NAME} in the file: ${DEST_SCRIPT_FILE} " - cat >> ${DEST_SCRIPT_FILE} <\n`date`\n" > ${HCOMPONENT_CONF_DIR}/ranger-security.xml - chown ${CFG_OWNER_INF} ${HCOMPONENT_CONF_DIR}/ranger-security.xml - chmod a+r ${HCOMPONENT_CONF_DIR}/ranger-security.xml - for cf in ${PROJ_INSTALL_DIR}/install/conf.templates/${action}/*.xml - do - cfb=`basename ${cf}` - if [ -f "${HCOMPONENT_CONF_DIR}/${cfb}" ] - then - log "Saving ${HCOMPONENT_CONF_DIR}/${cfb} to ${HCOMPONENT_CONF_DIR}/.${cfb}.${dt} ..." - cp ${HCOMPONENT_CONF_DIR}/${cfb} ${HCOMPONENT_CONF_DIR}/.${cfb}.${dt} - fi - cp ${cf} ${HCOMPONENT_CONF_DIR}/ - chown ${CFG_OWNER_INF} ${HCOMPONENT_CONF_DIR}/${cfb} - chmod a+r ${HCOMPONENT_CONF_DIR}/${cfb} - done - else - if [ -f ${HCOMPONENT_CONF_DIR}/ranger-security.xml ] - then - mv ${HCOMPONENT_CONF_DIR}/ranger-security.xml ${HCOMPONENT_CONF_DIR}/.ranger-security.xml.`date '+%Y%m%d%H%M%S'` - fi - fi - - # - # Ensure that POLICY_CACHE_FILE_PATH is accessible - # - REPO_NAME=$(getInstallProperty 'REPOSITORY_NAME') - export POLICY_CACHE_FILE_PATH=/etc/${PROJ_NAME}/${REPO_NAME}/policycache - export CREDENTIAL_PROVIDER_FILE=/etc/${PROJ_NAME}/${REPO_NAME}/cred.jceks - if [ ! -d ${POLICY_CACHE_FILE_PATH} ] - then - mkdir -p ${POLICY_CACHE_FILE_PATH} - fi - chmod a+rx /etc/${PROJ_NAME} - chmod a+rx /etc/${PROJ_NAME}/${REPO_NAME} - chmod a+rx ${POLICY_CACHE_FILE_PATH} - chown -R ${CFG_OWNER_INF} /etc/${PROJ_NAME}/${REPO_NAME} - - for f in ${PROJ_INSTALL_DIR}/install/conf.templates/${action}/*.cfg - do - if [ -f "${f}" ] - then - fn=`basename $f` - orgfn=`echo $fn | sed -e 's:-changes.cfg:.xml:'` - fullpathorgfn="${HCOMPONENT_CONF_DIR}/${orgfn}" - if [ ! -f ${fullpathorgfn} ] - then - if [ -f ${DEFAULT_XML_CONFIG} ] - then - log "Creating default file from [${DEFAULT_XML_CONFIG}] for [${fullpathorgfn}] .." - cp ${DEFAULT_XML_CONFIG} ${fullpathorgfn} - chown ${CFG_OWNER_INF} ${fullpathorgfn} - chmod a+r ${fullpathorgfn} - else - echo "ERROR: Unable to find ${fullpathorgfn}" - exit 1 - fi - fi - archivefn="${HCOMPONENT_CONF_DIR}/.${orgfn}.${dt}" - newfn="${HCOMPONENT_CONF_DIR}/.${orgfn}-new.${dt}" - log "Saving current config file: ${fullpathorgfn} to ${archivefn} ..." - cp ${fullpathorgfn} ${archivefn} - if [ $? -eq 0 ] - then - ${JAVA} -cp "${INSTALL_CP}" org.apache.ranger.utils.install.XmlConfigChanger -i ${archivefn} -o ${newfn} -c ${f} -p ${INSTALL_ARGS} - if [ $? -eq 0 ] - then - diff -w ${newfn} ${fullpathorgfn} > /dev/null 2>&1 - if [ $? -ne 0 ] - then - cat ${newfn} > ${fullpathorgfn} - fi - - # For Ambari install copy the .xml to conf.server also - if [ "${ambari_hive_install}" = "Y" ] - then - fullpathorgHS2fn="${HCOMPONENT_CONF_SERVER_DIR}/${orgfn}" - archiveHS2fn="${HCOMPONENT_CONF_SERVER_DIR}/.${orgfn}.${dt}" - newHS2fn="${HCOMPONENT_CONF_SERVER_DIR}/.${orgfn}-new.${dt}" - log "Saving current conf.server file: ${fullpathorgHS2fn} to ${archiveHS2fn} ..." - if [ -f ${fullpathorgHS2fn} ] - then - cp ${fullpathorgHS2fn} ${archiveHS2fn} - fi - cp ${fullpathorgfn} ${HCOMPONENT_CONF_SERVER_DIR}/${orgfn} - chown ${CFG_OWNER_INF} ${HCOMPONENT_CONF_SERVER_DIR}/${orgfn} - fi - - else - echo "ERROR: Unable to make changes to config. file: ${fullpathorgfn}" - echo "exiting ...." - exit 1 - fi - else - echo "ERROR: Unable to save config. file: ${fullpathorgfn} to ${archivefn}" - echo "exiting ...." - exit 1 - fi - fi - done - if [ "${HCOMPONENT_NAME}" = "hbase" ] || [ "${HCOMPONENT_NAME}" = "storm" ]; - then - chmod 644 ${HCOMPONENT_CONF_DIR}/* - fi -fi - -# -# Create library link -# -if [ "${action}" = "enable" ] -then - dt=`date '+%Y%m%d%H%M%S'` - for f in ${PROJ_LIB_DIR}/* - do - if [ -f "${f}" ] || [ -d "${f}" ] - then - bn=`basename $f` - if [ -f ${HCOMPONENT_LIB_DIR}/${bn} ] || [ -d ${HCOMPONENT_LIB_DIR}/${bn} ] - then - log "Saving lib file: ${HCOMPONENT_LIB_DIR}/${bn} to ${HCOMPONENT_LIB_DIR}/.${bn}.${dt} ..." - mv ${HCOMPONENT_LIB_DIR}/${bn} ${HCOMPONENT_LIB_DIR}/.${bn}.${dt} - fi - if [ ! -f ${HCOMPONENT_LIB_DIR}/${bn} ] && [ ! -d ${HCOMPONENT_LIB_DIR}/${bn} ] - then - ln -s ${f} ${HCOMPONENT_LIB_DIR}/${bn} - fi - fi - done - - # - # Encrypt the password and keep it secure in Credential Provider API - # - CredFile=${CREDENTIAL_PROVIDER_FILE} - if ! [ `echo ${CredFile} | grep '^/.*'` ] - then - echo "ERROR:Please enter the Credential File Store with proper file path" - exit 1 - fi - - pardir=`dirname ${CredFile}` - - if [ ! -d "${pardir}" ] - then - mkdir -p "${pardir}" - if [ $? -ne 0 ] - then - echo "ERROR: Unable to create credential store file path" - exit 1 - fi - chmod a+rx "${pardir}" - fi - - # - # Generate Credential Provider file and Credential for SSL KEYSTORE AND TRUSTSTORE - # - sslkeystoreAlias="sslKeyStore" - sslkeystoreCred=$(getInstallProperty 'SSL_KEYSTORE_PASSWORD') - create_jceks "${sslkeystoreAlias}" "${sslkeystoreCred}" "${CredFile}" - ssltruststoreAlias="sslTrustStore" - ssltruststoreCred=$(getInstallProperty 'SSL_TRUSTSTORE_PASSWORD') - create_jceks "${ssltruststoreAlias}" "${ssltruststoreCred}" "${CredFile}" - chown ${CFG_OWNER_INF} ${CredFile} - # - # To allow all users in the server (where Hive CLI and HBase CLI is used), - # user needs to have read access for the credential file. - # - chmod a+r ${CredFile} -fi - -# -# Knox specific configuration -# -# - -if [ "${HCOMPONENT_NAME}" = "knox" ] -then - if [ "${action}" = "enable" ] - then - authFrom="AclsAuthz" - authTo="XASecurePDPKnox" - else - authTo="AclsAuthz" - authFrom="XASecurePDPKnox" - fi - - dt=`date '+%Y%m%d%H%M%S'` - for fn in `ls ${HCOMPONENT_CONF_DIR}/topologies/*.xml 2> /dev/null` - do - if [ -f "${fn}" ] - then - dn=`dirname ${fn}` - bn=`basename ${fn}` - bf=${dn}/.${bn}.${dt} - echo "backup of ${fn} to ${bf} ..." - cp ${fn} ${bf} - echo "Updating topology file: [${fn}] ... " - cat ${fn} | sed -e "s-${authFrom}-${authTo}-" > ${fn}.${dt}.new - if [ $? -eq 0 ] - then - cat ${fn}.${dt}.new > ${fn} - rm ${fn}.${dt}.new - fi - fi - done -fi - -if [ "${HCOMPONENT_NAME}" = "storm" ] -then - CFG_FILE=${HCOMPONENT_CONF_DIR}/storm.yaml - ARCHIVE_FILE=${HCOMPONENT_CONF_DIR}/.storm.yaml.`date '+%Y%m%d%H%M%S'` - - if [ -f "${CFG_FILE}" ] - then - cp ${CFG_FILE} ${ARCHIVE_FILE} - - if [ "${action}" = "enable" ] - then - awk -F: 'BEGIN { - configured = 0 ; - } - { - if ($1 == "nimbus.authorizer") { - if ($2 ~ /^[ \t]*"org.apache.ranger.authorization.storm.authorizer.RangerStormAuthorizer"[ \t]*$/) { - configured = 1 ; - printf("%s\n",$0) ; - } - else { - printf("#%s\n",$0); - printf("nimbus.authorizer: \"org.apache.ranger.authorization.storm.authorizer.RangerStormAuthorizer\"\n") ; - configured = 1 ; - } - } - else { - printf("%s\n",$0) ; - } - } - END { - if (configured == 0) { - printf("nimbus.authorizer: \"org.apache.ranger.authorization.storm.authorizer.RangerStormAuthorizer\"\n") ; - } - }' ${CFG_FILE} > ${CFG_FILE}.new && cat ${CFG_FILE}.new > ${CFG_FILE} && rm -f ${CFG_FILE}.new - - else - awk -F: 'BEGIN { - configured = 0 ; - } - { - if ($1 == "nimbus.authorizer") { - if ($2 ~ /^[ \t]*"org.apache.storm.security.auth.authorizer.SimpleACLAuthorizer"[ \t]*$/) { - configured = 1 ; - printf("%s\n",$0) ; - } - else { - printf("#%s\n",$0); - printf("nimbus.authorizer: \"org.apache.storm.security.auth.authorizer.SimpleACLAuthorizer\"\n") ; - configured = 1 ; - } - } - else { - printf("%s\n",$0) ; - } - } - END { - if (configured == 0) { - printf("nimbus.authorizer: \"org.apache.storm.security.auth.authorizer.SimpleACLAuthorizer\"\n") ; - } - }' ${CFG_FILE} > ${CFG_FILE}.new && cat ${CFG_FILE}.new > ${CFG_FILE} && rm -f ${CFG_FILE}.new - fi - fi -fi - -#Check Properties whether in File, return code 1 if not exist -#$1 -> propertyName; $2 -> fileName -checkPropertyInFile(){ - validate=$(sed '/^\#/d' $2 | grep "^$1" | tail -n 1 | cut -d "=" -f1-) # for validation - if test -z "$validate" ; then return 1; fi -} - -#Add Properties to File -#$1 -> propertyName; $2 -> newPropertyValue; $3 -> fileName -addPropertyToFile(){ - echo "$1=$2">>$3 - validate=$(sed '/^\#/d' $3 | grep "^$1" | tail -n 1 | cut -d "=" -f2-) # for validation - if test -z "$validate" ; then log "[E] Failed to add properties '$1' to $3 file!"; exit 1; fi - echo "Property $1 added successfully with : '$2'" -} - -#Update Properties to File -#$1 -> propertyName; $2 -> newPropertyValue; $3 -> fileName -updatePropertyToFile(){ - sed 's@^'$1'=[^ ]*$@'$1'='$2'@g' $3 > /tmp/temporaryfile - cp /tmp/temporaryfile $3 - rm /tmp/temporaryfile - validate=$(sed '/^\#/d' $3 | grep "^$1" | tail -n 1 | cut -d "=" -f2-) # for validation - if test -z "$validate" ; then log "[E] '$1' not found in $3 file while Updating....!!"; exit 1; fi - echo "Property $1 updated successfully with : '$2'" -} - -#Add or Update Properties to File -#$1 -> propertyName; $2 -> newPropertyValue; $3 -> fileName -addOrUpdatePropertyToFile(){ - checkPropertyInFile $1 $3 - if [ $? -eq 1 ] - then - addPropertyToFile $1 $2 $3 - else - updatePropertyToFile $1 $2 $3 - fi -} - -if [ "${HCOMPONENT_NAME}" = "sqoop" ] -then - if [ "${action}" = "enable" ] - then - authName="org.apache.ranger.authorization.sqoop.authorizer.RangerSqoopAuthorizer" - else - authName="" - fi - - dt=`date '+%Y%m%d%H%M%S'` - fn=`ls ${HCOMPONENT_CONF_DIR}/sqoop.properties 2> /dev/null` - if [ -f "${fn}" ] - then - dn=`dirname ${fn}` - bn=`basename ${fn}` - bf=${dn}/.${bn}.${dt} - echo "backup of ${fn} to ${bf} ..." - cp ${fn} ${bf} - echo "Add or Update properties file: [${fn}] ... " - addOrUpdatePropertyToFile org.apache.sqoop.security.authorization.validator $authName ${fn} - fi -fi - -if [ "${HCOMPONENT_NAME}" = "kylin" ] -then - if [ "${action}" = "enable" ] - then - authName="org.apache.ranger.authorization.kylin.authorizer.RangerKylinAuthorizer" - else - authName="" - fi - - dt=`date '+%Y%m%d%H%M%S'` - fn=`ls ${HCOMPONENT_CONF_DIR}/kylin.properties 2> /dev/null` - if [ -f "${fn}" ] - then - dn=`dirname ${fn}` - bn=`basename ${fn}` - bf=${dn}/.${bn}.${dt} - echo "backup of ${fn} to ${bf} ..." - cp ${fn} ${bf} - echo "Add or Update properties file: [${fn}] ... " - addOrUpdatePropertyToFile kylin.server.external-acl-provider $authName ${fn} - fi -fi - -if [ "${HCOMPONENT_NAME}" = "presto" ] -then - if [ "${action}" = "enable" ] - then - controlName="ranger" - else - controlName="" - fi - dt=`date '+%Y%m%d%H%M%S'` - fn=`ls ${HCOMPONENT_CONF_DIR}/access-control.properties 2> /dev/null` - if [ -f "${fn}" ] - then - dn=`dirname ${fn}` - bn=`basename ${fn}` - bf=${dn}/.${bn}.${dt} - echo "backup of ${fn} to ${bf} ..." - cp ${fn} ${bf} - else - fn=${HCOMPONENT_CONF_DIR}/access-control.properties - fi - echo "Add or Update properties file: [${fn}] ... " - addOrUpdatePropertyToFile access-control.name $controlName ${fn} - echo "Linking config files" - cd ${HCOMPONENT_LIB_DIR}/ranger-presto-plugin-impl/ - ln -sf ${HCOMPONENT_CONF_DIR} conf -fi - - -# -# Set notice to restart the ${HCOMPONENT_NAME} -# - -echo "Ranger Plugin for ${HCOMPONENT_NAME} has been ${action}d. Please restart ${HCOMPONENT_NAME} to ensure that changes are effective." - -exit 0 diff --git a/atlas-hub/pre-conf/ranger/install.properties b/atlas-hub/pre-conf/ranger/install.properties deleted file mode 100755 index 8cf19fc3b8..0000000000 --- a/atlas-hub/pre-conf/ranger/install.properties +++ /dev/null @@ -1,106 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# -# Location of Policy Manager URL -# -# Example: -# POLICY_MGR_URL=http://policymanager.xasecure.net:6080 -# -POLICY_MGR_URL=RANGER_SERVICE_URL - -# -# This is the repository name created within policy manager -# -# Example: -# REPOSITORY_NAME=atlasdev -# -REPOSITORY_NAME=ATLAS_REPOSITORY_NAME - -# -# Atlas installation directory -# -COMPONENT_INSTALL_DIR_NAME=ATLAS_INSTALL_DIR - -# AUDIT configuration with V3 properties - -#Should audit be summarized at source -XAAUDIT.SUMMARY.ENABLE=true - -# Enable audit logs to Solr -#Example -#XAAUDIT.SOLR.ENABLE=true -#XAAUDIT.SOLR.URL=http://localhost:6083/solr/ranger_audits -#XAAUDIT.SOLR.ZOOKEEPER= -#XAAUDIT.SOLR.FILE_SPOOL_DIR=/var/log/atlas/audit/solr/spool - -XAAUDIT.SOLR.ENABLE=false -XAAUDIT.SOLR.URL=NONE -XAAUDIT.SOLR.USER=NONE -XAAUDIT.SOLR.PASSWORD=NONE -XAAUDIT.SOLR.ZOOKEEPER=NONE - -# Enable audit logs to HDFS -#Example -#XAAUDIT.HDFS.ENABLE=true -#XAAUDIT.HDFS.HDFS_DIR=hdfs://node-1.example.com:8020/ranger/audit -# If using Azure Blob Storage -#XAAUDIT.HDFS.HDFS_DIR=wasb[s]://@.blob.core.windows.net/ -#XAAUDIT.HDFS.HDFS_DIR=wasb://ranger_audit_container@my-azure-account.blob.core.windows.net/ranger/audit -#XAAUDIT.HDFS.FILE_SPOOL_DIR=/var/log/atlas/audit/hdfs/spool - -XAAUDIT.HDFS.ENABLE=false -XAAUDIT.HDFS.HDFS_DIR=hdfs://__REPLACE__NAME_NODE_HOST:8020/ranger/audit - -# Following additional propertis are needed When auditing to Azure Blob Storage via HDFS -# Get these values from your /etc/hadoop/conf/core-site.xml -#XAAUDIT.HDFS.HDFS_DIR=wasb[s]://@.blob.core.windows.net/ -XAAUDIT.HDFS.AZURE_ACCOUNTNAME=__REPLACE_AZURE_ACCOUNT_NAME -XAAUDIT.HDFS.AZURE_ACCOUNTKEY=__REPLACE_AZURE_ACCOUNT_KEY -XAAUDIT.HDFS.AZURE_SHELL_KEY_PROVIDER=__REPLACE_AZURE_SHELL_KEY_PROVIDER -XAAUDIT.HDFS.AZURE_ACCOUNTKEY_PROVIDER=__REPLACE_AZURE_ACCOUNT_KEY_PROVIDER - -# End of V3 properties - - -# -# SSL Client Certificate Information -# -# Example: -# SSL_KEYSTORE_FILE_PATH=/etc/atlas/conf/ranger-plugin-keystore.jks -# SSL_KEYSTORE_PASSWORD=none -# SSL_TRUSTSTORE_FILE_PATH=/etc/atlas/conf/ranger-plugin-truststore.jks -# SSL_TRUSTSTORE_PASSWORD=none -# -# You do not need use SSL between agent and security admin tool, please leave these sample value as it is. -# -SSL_KEYSTORE_FILE_PATH=/etc/atlas/conf/ranger-plugin-keystore.jks -SSL_KEYSTORE_PASSWORD=myKeyFilePassword -SSL_TRUSTSTORE_FILE_PATH=/etc/atlas/conf/ranger-plugin-truststore.jks -SSL_TRUSTSTORE_PASSWORD=changeit - -# -# Custom component user -# CUSTOM_COMPONENT_USER= -# keep blank if component user is default -CUSTOM_USER=atlas - - -# -# Custom component group -# CUSTOM_COMPONENT_GROUP= -# keep blank if component group is default -CUSTOM_GROUP=hadoop - diff --git a/atlas-hub/pre-conf/ranger/install/conf.templates/default/configuration.xml b/atlas-hub/pre-conf/ranger/install/conf.templates/default/configuration.xml deleted file mode 100755 index bdb0125891..0000000000 --- a/atlas-hub/pre-conf/ranger/install/conf.templates/default/configuration.xml +++ /dev/null @@ -1,20 +0,0 @@ - - - - - diff --git a/atlas-hub/pre-conf/ranger/install/conf.templates/enable/ranger-atlas-audit-changes.cfg b/atlas-hub/pre-conf/ranger/install/conf.templates/enable/ranger-atlas-audit-changes.cfg deleted file mode 100755 index 07fc382798..0000000000 --- a/atlas-hub/pre-conf/ranger/install/conf.templates/enable/ranger-atlas-audit-changes.cfg +++ /dev/null @@ -1,30 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -#solr configuration -xasecure.audit.destination.solr %XAAUDIT.SOLR.ENABLE% mod create-if-not-exists -xasecure.audit.destination.solr.urls %XAAUDIT.SOLR.URL% mod create-if-not-exists -xasecure.audit.destination.solr.user %XAAUDIT.SOLR.USER% mod create-if-not-exists -xasecure.audit.destination.solr.password %XAAUDIT.SOLR.PASSWORD% mod create-if-not-exists -xasecure.audit.destination.solr.zookeepers %XAAUDIT.SOLR.ZOOKEEPER% mod create-if-not-exists - - -#hdfs configuration -xasecure.audit.destination.hdfs %XAAUDIT.HDFS.ENABLE% mod create-if-not-exists -xasecure.audit.destination.hdfs.dir %XAAUDIT.HDFS.HDFS_DIR% mod create-if-not-exists - - - diff --git a/atlas-hub/pre-conf/ranger/install/conf.templates/enable/ranger-atlas-audit.xml b/atlas-hub/pre-conf/ranger/install/conf.templates/enable/ranger-atlas-audit.xml deleted file mode 100755 index ca1dd338f2..0000000000 --- a/atlas-hub/pre-conf/ranger/install/conf.templates/enable/ranger-atlas-audit.xml +++ /dev/null @@ -1,139 +0,0 @@ - - - - - - xasecure.audit.is.enabled - true - - - - - - xasecure.audit.destination.solr - false - - - - xasecure.audit.destination.solr.urls - NONE - - - - xasecure.audit.destination.solr.zookeepers - - - - - xasecure.audit.destination.solr.collection - NONE - - - - - - xasecure.audit.destination.hdfs - false - - - - xasecure.audit.destination.hdfs.dir - hdfs://__REPLACE__NAME_NODE_HOST:8020/ranger/audit - - - - xasecure.audit.destination.hdfs.subdir - %app-type%/%time:yyyyMMdd% - - - - xasecure.audit.destination.hdfs.filename.format - %app-type%_ranger_audit_%hostname%.log - - - - xasecure.audit.destination.hdfs.file.rollover.sec - 86400 - - - - - - - xasecure.audit.destination.log4j - true - - - - xasecure.audit.destination.log4j.logger - AUDIT - - - - - xasecure.audit.destination.elasticsearch - true - - - xasecure.audit.elasticsearch.is.enabled - true/ - - - xasecure.audit.destination.elasticsearch.urls - logging-master.logging.svc.cluster.local - - - xasecure.audit.destination.elasticsearch.index - ranger-audit - - diff --git a/atlas-hub/pre-conf/ranger/install/conf.templates/enable/ranger-atlas-security-changes.cfg b/atlas-hub/pre-conf/ranger/install/conf.templates/enable/ranger-atlas-security-changes.cfg deleted file mode 100755 index 08790b9852..0000000000 --- a/atlas-hub/pre-conf/ranger/install/conf.templates/enable/ranger-atlas-security-changes.cfg +++ /dev/null @@ -1,29 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Change the original policy parameter to work with policy manager based. -# -# -ranger.plugin.atlas.service.name %REPOSITORY_NAME% mod create-if-not-exists - -ranger.plugin.atlas.policy.source.impl org.apache.ranger.admin.client.RangerAdminRESTClient mod create-if-not-exists - -ranger.plugin.atlas.policy.rest.url %POLICY_MGR_URL% mod create-if-not-exists -ranger.plugin.atlas.policy.rest.ssl.config.file /etc/atlas/conf/ranger-policymgr-ssl.xml mod create-if-not-exists -ranger.plugin.atlas.policy.pollIntervalMs 30000 mod create-if-not-exists -ranger.plugin.atlas.policy.cache.dir %POLICY_CACHE_FILE_PATH% mod create-if-not-exists -ranger.plugin.atlas.policy.rest.client.connection.timeoutMs 120000 mod create-if-not-exists -ranger.plugin.atlas.policy.rest.client.read.timeoutMs 30000 mod create-if-not-exists - diff --git a/atlas-hub/pre-conf/ranger/install/conf.templates/enable/ranger-atlas-security.xml b/atlas-hub/pre-conf/ranger/install/conf.templates/enable/ranger-atlas-security.xml deleted file mode 100755 index 933c8e78df..0000000000 --- a/atlas-hub/pre-conf/ranger/install/conf.templates/enable/ranger-atlas-security.xml +++ /dev/null @@ -1,83 +0,0 @@ - - - - - - ranger.plugin.atlas.service.name - atlasdev - - Name of the Ranger service containing policies for this YARN instance - - - - - ranger.plugin.atlas.policy.source.impl - org.apache.ranger.admin.client.RangerAdminRESTClient - - Class to retrieve policies from the source - - - - - ranger.plugin.atlas.policy.rest.url - http://policymanagerhost:port - - URL to Ranger Admin - - - - - ranger.plugin.atlas.policy.rest.ssl.config.file - /etc/atlas/conf/ranger-policymgr-ssl.xml - - Path to the file containing SSL details to contact Ranger Admin - - - - - ranger.plugin.atlas.policy.pollIntervalMs - 30000 - - How often to poll for changes in policies? - - - - - ranger.plugin.atlas.policy.cache.dir - /etc/ranger/atlasdev/policycache - - Directory where Ranger policies are cached after successful retrieval from the source - - - - - ranger.plugin.atlas.policy.rest.client.connection.timeoutMs - 120000 - - RangerRestClient Connection Timeout in Milli Seconds - - - - - ranger.plugin.atlas.policy.rest.client.read.timeoutMs - 30000 - - RangerRestClient read Timeout in Milli Seconds - - - diff --git a/atlas-hub/pre-conf/ranger/install/conf.templates/enable/ranger-policymgr-ssl-changes.cfg b/atlas-hub/pre-conf/ranger/install/conf.templates/enable/ranger-policymgr-ssl-changes.cfg deleted file mode 100755 index 47126492f2..0000000000 --- a/atlas-hub/pre-conf/ranger/install/conf.templates/enable/ranger-policymgr-ssl-changes.cfg +++ /dev/null @@ -1,21 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SSL Params -# -xasecure.policymgr.clientssl.keystore %SSL_KEYSTORE_FILE_PATH% mod create-if-not-exists -xasecure.policymgr.clientssl.keystore.credential.file jceks://file%CREDENTIAL_PROVIDER_FILE% mod create-if-not-exists -xasecure.policymgr.clientssl.truststore %SSL_TRUSTSTORE_FILE_PATH% mod create-if-not-exists -xasecure.policymgr.clientssl.truststore.credential.file jceks://file%CREDENTIAL_PROVIDER_FILE% mod create-if-not-exists \ No newline at end of file diff --git a/atlas-hub/pre-conf/ranger/install/conf.templates/enable/ranger-policymgr-ssl.xml b/atlas-hub/pre-conf/ranger/install/conf.templates/enable/ranger-policymgr-ssl.xml deleted file mode 100755 index 3baf7725cf..0000000000 --- a/atlas-hub/pre-conf/ranger/install/conf.templates/enable/ranger-policymgr-ssl.xml +++ /dev/null @@ -1,49 +0,0 @@ - - - - - - - xasecure.policymgr.clientssl.keystore - hadoopdev-clientcert.jks - - Java Keystore files - - - - xasecure.policymgr.clientssl.truststore - cacerts-xasecure.jks - - java truststore file - - - - xasecure.policymgr.clientssl.keystore.credential.file - jceks://file/tmp/keystore-hadoopdev-ssl.jceks - - java keystore credential file - - - - xasecure.policymgr.clientssl.truststore.credential.file - jceks://file/tmp/truststore-hadoopdev-ssl.jceks - - java truststore credential file - - - \ No newline at end of file diff --git a/atlas-hub/pre-conf/ranger/install/conf.templates/enable/ranger-security.xml b/atlas-hub/pre-conf/ranger/install/conf.templates/enable/ranger-security.xml deleted file mode 100644 index 1a7cb6fe0e..0000000000 --- a/atlas-hub/pre-conf/ranger/install/conf.templates/enable/ranger-security.xml +++ /dev/null @@ -1,17 +0,0 @@ - -\nThu Apr 30 22:10:46 UTC 2020\n \ No newline at end of file diff --git a/atlas-hub/pre-conf/ranger/install/lib/commons-cli-1.2.jar b/atlas-hub/pre-conf/ranger/install/lib/commons-cli-1.2.jar deleted file mode 100644 index ce4b9fffe4..0000000000 Binary files a/atlas-hub/pre-conf/ranger/install/lib/commons-cli-1.2.jar and /dev/null differ diff --git a/atlas-hub/pre-conf/ranger/install/lib/commons-collections-3.2.2.jar b/atlas-hub/pre-conf/ranger/install/lib/commons-collections-3.2.2.jar deleted file mode 100644 index fa5df82a63..0000000000 Binary files a/atlas-hub/pre-conf/ranger/install/lib/commons-collections-3.2.2.jar and /dev/null differ diff --git a/atlas-hub/pre-conf/ranger/install/lib/commons-configuration2-2.1.1.jar b/atlas-hub/pre-conf/ranger/install/lib/commons-configuration2-2.1.1.jar deleted file mode 100644 index 666baa09d2..0000000000 Binary files a/atlas-hub/pre-conf/ranger/install/lib/commons-configuration2-2.1.1.jar and /dev/null differ diff --git a/atlas-hub/pre-conf/ranger/install/lib/commons-io-2.5.jar b/atlas-hub/pre-conf/ranger/install/lib/commons-io-2.5.jar deleted file mode 100644 index 107b061f5f..0000000000 Binary files a/atlas-hub/pre-conf/ranger/install/lib/commons-io-2.5.jar and /dev/null differ diff --git a/atlas-hub/pre-conf/ranger/install/lib/commons-lang-2.6.jar b/atlas-hub/pre-conf/ranger/install/lib/commons-lang-2.6.jar deleted file mode 100644 index 98467d3a65..0000000000 Binary files a/atlas-hub/pre-conf/ranger/install/lib/commons-lang-2.6.jar and /dev/null differ diff --git a/atlas-hub/pre-conf/ranger/install/lib/commons-logging-1.2.jar b/atlas-hub/pre-conf/ranger/install/lib/commons-logging-1.2.jar deleted file mode 100644 index 93a3b9f6db..0000000000 Binary files a/atlas-hub/pre-conf/ranger/install/lib/commons-logging-1.2.jar and /dev/null differ diff --git a/atlas-hub/pre-conf/ranger/install/lib/credentialbuilder-2.0.0.jar b/atlas-hub/pre-conf/ranger/install/lib/credentialbuilder-2.0.0.jar deleted file mode 100644 index b082d9bcea..0000000000 Binary files a/atlas-hub/pre-conf/ranger/install/lib/credentialbuilder-2.0.0.jar and /dev/null differ diff --git a/atlas-hub/pre-conf/ranger/install/lib/guava-25.1-jre.jar b/atlas-hub/pre-conf/ranger/install/lib/guava-25.1-jre.jar deleted file mode 100644 index babc175535..0000000000 Binary files a/atlas-hub/pre-conf/ranger/install/lib/guava-25.1-jre.jar and /dev/null differ diff --git a/atlas-hub/pre-conf/ranger/install/lib/hadoop-auth-3.1.1.jar b/atlas-hub/pre-conf/ranger/install/lib/hadoop-auth-3.1.1.jar deleted file mode 100644 index 0d2a9dcd3f..0000000000 Binary files a/atlas-hub/pre-conf/ranger/install/lib/hadoop-auth-3.1.1.jar and /dev/null differ diff --git a/atlas-hub/pre-conf/ranger/install/lib/hadoop-common-3.1.1.jar b/atlas-hub/pre-conf/ranger/install/lib/hadoop-common-3.1.1.jar deleted file mode 100644 index 0bbed5ac7e..0000000000 Binary files a/atlas-hub/pre-conf/ranger/install/lib/hadoop-common-3.1.1.jar and /dev/null differ diff --git a/atlas-hub/pre-conf/ranger/install/lib/htrace-core4-4.1.0-incubating.jar b/atlas-hub/pre-conf/ranger/install/lib/htrace-core4-4.1.0-incubating.jar deleted file mode 100644 index 12349a2066..0000000000 Binary files a/atlas-hub/pre-conf/ranger/install/lib/htrace-core4-4.1.0-incubating.jar and /dev/null differ diff --git a/atlas-hub/pre-conf/ranger/install/lib/ranger-plugins-cred-2.0.0.jar b/atlas-hub/pre-conf/ranger/install/lib/ranger-plugins-cred-2.0.0.jar deleted file mode 100644 index 01653b1fc5..0000000000 Binary files a/atlas-hub/pre-conf/ranger/install/lib/ranger-plugins-cred-2.0.0.jar and /dev/null differ diff --git a/atlas-hub/pre-conf/ranger/install/lib/ranger-plugins-installer-2.0.0.jar b/atlas-hub/pre-conf/ranger/install/lib/ranger-plugins-installer-2.0.0.jar deleted file mode 100644 index ab774b9329..0000000000 Binary files a/atlas-hub/pre-conf/ranger/install/lib/ranger-plugins-installer-2.0.0.jar and /dev/null differ diff --git a/atlas-hub/pre-conf/ranger/install/lib/slf4j-api-1.7.30.jar b/atlas-hub/pre-conf/ranger/install/lib/slf4j-api-1.7.30.jar deleted file mode 100644 index 29ac26fb8c..0000000000 Binary files a/atlas-hub/pre-conf/ranger/install/lib/slf4j-api-1.7.30.jar and /dev/null differ diff --git a/atlas-hub/pre-conf/ranger/install/lib/stax2-api-3.1.4.jar b/atlas-hub/pre-conf/ranger/install/lib/stax2-api-3.1.4.jar deleted file mode 100644 index dded036928..0000000000 Binary files a/atlas-hub/pre-conf/ranger/install/lib/stax2-api-3.1.4.jar and /dev/null differ diff --git a/atlas-hub/pre-conf/ranger/install/lib/woodstox-core-5.0.3.jar b/atlas-hub/pre-conf/ranger/install/lib/woodstox-core-5.0.3.jar deleted file mode 100644 index 1c268641c8..0000000000 Binary files a/atlas-hub/pre-conf/ranger/install/lib/woodstox-core-5.0.3.jar and /dev/null differ diff --git a/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-impl/eclipselink-2.5.2.jar b/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-impl/eclipselink-2.5.2.jar deleted file mode 100644 index 325e2a3f90..0000000000 Binary files a/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-impl/eclipselink-2.5.2.jar and /dev/null differ diff --git a/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-impl/gethostname4j-0.0.2.jar b/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-impl/gethostname4j-0.0.2.jar deleted file mode 100644 index 507d8d7b4c..0000000000 Binary files a/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-impl/gethostname4j-0.0.2.jar and /dev/null differ diff --git a/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-impl/guava-25.1-jre.jar b/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-impl/guava-25.1-jre.jar deleted file mode 100644 index babc175535..0000000000 Binary files a/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-impl/guava-25.1-jre.jar and /dev/null differ diff --git a/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-impl/jackson-jaxrs-1.9.13.jar b/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-impl/jackson-jaxrs-1.9.13.jar deleted file mode 100644 index b85f45cf40..0000000000 Binary files a/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-impl/jackson-jaxrs-1.9.13.jar and /dev/null differ diff --git a/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-impl/javax.persistence-2.1.0.jar b/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-impl/javax.persistence-2.1.0.jar deleted file mode 100644 index e48d2e9618..0000000000 Binary files a/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-impl/javax.persistence-2.1.0.jar and /dev/null differ diff --git a/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-impl/jersey-bundle-1.19.3.jar b/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-impl/jersey-bundle-1.19.3.jar deleted file mode 100644 index bd786a7821..0000000000 Binary files a/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-impl/jersey-bundle-1.19.3.jar and /dev/null differ diff --git a/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-impl/jna-5.2.0.jar b/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-impl/jna-5.2.0.jar deleted file mode 100644 index 0b5fabdd8b..0000000000 Binary files a/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-impl/jna-5.2.0.jar and /dev/null differ diff --git a/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-impl/jna-platform-5.2.0.jar b/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-impl/jna-platform-5.2.0.jar deleted file mode 100644 index 4722681821..0000000000 Binary files a/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-impl/jna-platform-5.2.0.jar and /dev/null differ diff --git a/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-impl/ranger-atlas-plugin-2.2.0.jar b/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-impl/ranger-atlas-plugin-2.2.0.jar deleted file mode 100644 index 178b6191eb..0000000000 Binary files a/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-impl/ranger-atlas-plugin-2.2.0.jar and /dev/null differ diff --git a/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-impl/ranger-plugins-audit-2.2.0.jar b/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-impl/ranger-plugins-audit-2.2.0.jar deleted file mode 100644 index e108d19e1d..0000000000 Binary files a/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-impl/ranger-plugins-audit-2.2.0.jar and /dev/null differ diff --git a/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-impl/ranger-plugins-common-2.2.0.jar b/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-impl/ranger-plugins-common-2.2.0.jar deleted file mode 100644 index cc23b8a0b2..0000000000 Binary files a/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-impl/ranger-plugins-common-2.2.0.jar and /dev/null differ diff --git a/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-impl/ranger-plugins-cred-2.2.0-SNAPSHOT.jar b/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-impl/ranger-plugins-cred-2.2.0-SNAPSHOT.jar deleted file mode 100644 index d270f48d96..0000000000 Binary files a/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-impl/ranger-plugins-cred-2.2.0-SNAPSHOT.jar and /dev/null differ diff --git a/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-impl/solr-solrj-7.7.1.jar b/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-impl/solr-solrj-7.7.1.jar deleted file mode 100644 index 0cd9fd57ba..0000000000 Binary files a/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-impl/solr-solrj-7.7.1.jar and /dev/null differ diff --git a/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-shim-3.0.0-SNAPSHOT.jar b/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-shim-3.0.0-SNAPSHOT.jar deleted file mode 100644 index 89893ad6ce..0000000000 Binary files a/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-shim-3.0.0-SNAPSHOT.jar and /dev/null differ diff --git a/atlas-hub/pre-conf/ranger/lib/ranger-plugin-classloader-3.0.0-SNAPSHOT.jar b/atlas-hub/pre-conf/ranger/lib/ranger-plugin-classloader-3.0.0-SNAPSHOT.jar deleted file mode 100644 index 866c8a72b6..0000000000 Binary files a/atlas-hub/pre-conf/ranger/lib/ranger-plugin-classloader-3.0.0-SNAPSHOT.jar and /dev/null differ diff --git a/atlas-hub/pre-conf/ranger/ranger_credential_helper.py b/atlas-hub/pre-conf/ranger/ranger_credential_helper.py deleted file mode 100755 index b36adeace1..0000000000 --- a/atlas-hub/pre-conf/ranger/ranger_credential_helper.py +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/python -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import sys -import os -from subprocess import Popen,PIPE -from optparse import OptionParser - -if os.getenv('JAVA_HOME') is None: - print "ERROR: JAVA_HOME environment property was not defined, exit." - sys.exit(1) -else: - JAVA_BIN=os.path.join(os.getenv('JAVA_HOME'),'bin','java') -print "Using Java:" + str(JAVA_BIN) - -def main(): - - parser = OptionParser() - - parser.add_option("-l", "--libpath", dest="library_path", help="Path to folder where credential libs are present") - parser.add_option("-f", "--file", dest="jceks_file_path", help="Path to jceks file to use") - parser.add_option("-k", "--key", dest="key", help="Key to use") - parser.add_option("-v", "--value", dest="value", help="Value to use") - parser.add_option("-c", "--create", dest="create", help="Add a new alias") - - (options, args) = parser.parse_args() - library_path = options.library_path - jceks_file_path = options.jceks_file_path - key = options.key - value = options.value - getorcreate = 'create' if options.create else 'get' - call_keystore(library_path, jceks_file_path, key, value, getorcreate) - - -def call_keystore(libpath, filepath, aliasKey, aliasValue='', getorcreate='get'): - finalLibPath = libpath.replace('\\','/').replace('//','/') - finalFilePath = 'jceks://file/'+filepath.replace('\\','/').replace('//','/') - if getorcreate == 'create': - commandtorun = [JAVA_BIN, '-cp', finalLibPath, 'org.apache.ranger.credentialapi.buildks' ,'create', aliasKey, '-value', aliasValue, '-provider',finalFilePath] - p = Popen(commandtorun,stdin=PIPE, stdout=PIPE, stderr=PIPE) - output, error = p.communicate() - statuscode = p.returncode - if statuscode == 0: - print "Alias " + aliasKey + " created successfully!" - else : - print "Error creating Alias!! Error: " + str(error) - - elif getorcreate == 'get': - commandtorun = [JAVA_BIN, '-cp', finalLibPath, 'org.apache.ranger.credentialapi.buildks' ,'get', aliasKey, '-provider',finalFilePath] - p = Popen(commandtorun,stdin=PIPE, stdout=PIPE, stderr=PIPE) - output, error = p.communicate() - statuscode = p.returncode - if statuscode == 0: - print "Alias : " + aliasKey + " Value : " + str(output) - else : - print "Error getting value!! Error: " + str(error) - - else: - print 'Invalid Arguments!!' - -if __name__ == '__main__': - main() diff --git a/atlas-hub/pre-conf/solr/currency.xml b/atlas-hub/pre-conf/solr/currency.xml deleted file mode 100644 index 3a9c58afee..0000000000 --- a/atlas-hub/pre-conf/solr/currency.xml +++ /dev/null @@ -1,67 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/atlas-hub/pre-conf/solr/lang/stopwords_en.txt b/atlas-hub/pre-conf/solr/lang/stopwords_en.txt deleted file mode 100644 index 2c164c0b2a..0000000000 --- a/atlas-hub/pre-conf/solr/lang/stopwords_en.txt +++ /dev/null @@ -1,54 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# a couple of test stopwords to test that the words are really being -# configured from this file: -stopworda -stopwordb - -# Standard english stop words taken from Lucene's StopAnalyzer -a -an -and -are -as -at -be -but -by -for -if -in -into -is -it -no -not -of -on -or -such -that -the -their -then -there -these -they -this -to -was -will -with diff --git a/atlas-hub/pre-conf/solr/protwords.txt b/atlas-hub/pre-conf/solr/protwords.txt deleted file mode 100644 index 1dfc0abecb..0000000000 --- a/atlas-hub/pre-conf/solr/protwords.txt +++ /dev/null @@ -1,21 +0,0 @@ -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -#----------------------------------------------------------------------- -# Use a protected word file to protect against the stemmer reducing two -# unrelated words to the same base word. - -# Some non-words that normally won't be encountered, -# just to test that they won't be stemmed. -dontstems -zwhacky - diff --git a/atlas-hub/pre-conf/solr/schema.xml b/atlas-hub/pre-conf/solr/schema.xml deleted file mode 100644 index 1d445b1500..0000000000 --- a/atlas-hub/pre-conf/solr/schema.xml +++ /dev/null @@ -1,534 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - id - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/atlas-hub/pre-conf/solr/solrconfig.xml b/atlas-hub/pre-conf/solr/solrconfig.xml deleted file mode 100644 index 1d414f76e9..0000000000 --- a/atlas-hub/pre-conf/solr/solrconfig.xml +++ /dev/null @@ -1,625 +0,0 @@ - - - - - - - - - 5.0.0 - - - ${solr.data.dir:} - - - - - - - - ${solr.hdfs.home:} - - ${solr.hdfs.confdir:} - - ${solr.hdfs.blockcache.enabled:true} - - ${solr.hdfs.blockcache.global:true} - - - - - - - - - - - - - - - - ${solr.lock.type:native} - - - true - - - - - - - - - - - - - - - - ${solr.ulog.dir:} - - - - - ${solr.autoCommit.maxTime:15000} - false - - - - - ${solr.autoSoftCommit.maxTime:-1} - - - - - - - - 1024 - - - - - - - - - - - - - - - - - - true - - - 20 - - - 200 - - - false - - - 2 - - - - - - - - - - - - - - - - - - - - explicit - 10 - - - - - - - - explicit - json - true - text - - - - - - - {!xport} - xsort - false - - - - query - - - - - - - text - - - - - - - - - - - - - - explicit - true - - - - - - - - - - - - - - true - false - - - terms - - - - - - *:* - - - - - - - timestamp - - - 5 - ttl - expire_at - - - expire_at_dt - - - - - diff --git a/atlas-hub/pre-conf/solr/stopwords.txt b/atlas-hub/pre-conf/solr/stopwords.txt deleted file mode 100644 index ae1e83eeb3..0000000000 --- a/atlas-hub/pre-conf/solr/stopwords.txt +++ /dev/null @@ -1,14 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/atlas-hub/pre-conf/solr/synonyms.txt b/atlas-hub/pre-conf/solr/synonyms.txt deleted file mode 100644 index 7f72128303..0000000000 --- a/atlas-hub/pre-conf/solr/synonyms.txt +++ /dev/null @@ -1,29 +0,0 @@ -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -#----------------------------------------------------------------------- -#some test synonym mappings unlikely to appear in real input text -aaafoo => aaabar -bbbfoo => bbbfoo bbbbar -cccfoo => cccbar cccbaz -fooaaa,baraaa,bazaaa - -# Some synonym groups specific to this example -GB,gib,gigabyte,gigabytes -MB,mib,megabyte,megabytes -Television, Televisions, TV, TVs -#notice we use "gib" instead of "GiB" so any WordDelimiterFilter coming -#after us won't split it into two words. - -# Synonym mappings can be used for spelling correction too -pixima => pixma - diff --git a/auth-audits/pom.xml b/auth-audits/pom.xml index c19e686740..60037a3183 100644 --- a/auth-audits/pom.xml +++ b/auth-audits/pom.xml @@ -48,46 +48,6 @@ ${commons-lang.version} - - org.apache.solr - solr-solrj - ${solr.version} - - - io.netty - * - - - org.eclipse.jetty.http2 - * - - - org.apache.commons - commons-math3 - - - commons-io - commons-io - - - org.apache.httpcomponents - * - - - org.apache.zookeeper - * - - - org.codehaus.woodstox - * - - - org.eclipse.jetty - * - - - - org.elasticsearch.client elasticsearch-rest-high-level-client diff --git a/auth-audits/src/main/java/org/apache/atlas/audit/destination/SolrAuditDestination.java b/auth-audits/src/main/java/org/apache/atlas/audit/destination/SolrAuditDestination.java deleted file mode 100644 index 7b42926ca9..0000000000 --- a/auth-audits/src/main/java/org/apache/atlas/audit/destination/SolrAuditDestination.java +++ /dev/null @@ -1,489 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.atlas.audit.destination; - -import org.apache.commons.lang.StringUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.atlas.audit.model.AuditEventBase; -import org.apache.atlas.audit.model.AuthzAuditEvent; -import org.apache.atlas.audit.provider.MiscUtil; -import org.apache.atlas.audit.utils.InMemoryJAASConfiguration; -import org.apache.atlas.audit.utils.KerberosAction; -import org.apache.atlas.audit.utils.KerberosJAASConfigUser; -import org.apache.atlas.audit.utils.KerberosUser; -import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.impl.CloudSolrClient; -import org.apache.solr.client.solrj.impl.HttpClientUtil; -import org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder; -import org.apache.solr.client.solrj.impl.LBHttpSolrClient; -import org.apache.solr.client.solrj.impl.SolrHttpClientBuilder; -import org.apache.solr.client.solrj.response.UpdateResponse; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.SolrInputDocument; - -import javax.net.ssl.KeyManager; -import javax.net.ssl.KeyManagerFactory; -import javax.net.ssl.SSLContext; -import javax.net.ssl.TrustManager; -import javax.net.ssl.TrustManagerFactory; -import javax.security.auth.login.LoginException; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.security.KeyManagementException; -import java.security.KeyStore; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; -import java.security.PrivilegedExceptionAction; -import java.security.SecureRandom; -import java.security.UnrecoverableKeyException; -import java.security.cert.CertificateException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.List; -import java.util.Optional; -import java.util.Properties; - - -public class SolrAuditDestination extends AuditDestination { - private static final Log LOG = LogFactory - .getLog(SolrAuditDestination.class); - - public static final String PROP_SOLR_URLS = "urls"; - public static final String PROP_SOLR_ZK = "zookeepers"; - public static final String PROP_SOLR_COLLECTION = "collection"; - public static final String PROP_SOLR_FORCE_USE_INMEMORY_JAAS_CONFIG = "force.use.inmemory.jaas.config"; - - public static final String DEFAULT_COLLECTION_NAME = "ranger_audits"; - public static final String PROP_JAVA_SECURITY_AUTH_LOGIN_CONFIG = "java.security.auth.login.config"; - - private volatile SolrClient solrClient = null; - private volatile KerberosUser kerberosUser = null; - - public SolrAuditDestination() { - } - - @Override - public void init(Properties props, String propPrefix) { - LOG.info("init() called"); - super.init(props, propPrefix); - init(); - connect(); - } - - @Override - public void stop() { - LOG.info("SolrAuditDestination.stop() called.."); - logStatus(); - - if (solrClient != null) { - try { - solrClient.close(); - } catch (IOException ioe) { - LOG.error("Error while stopping slor!", ioe); - } finally { - solrClient = null; - } - } - - if (kerberosUser != null) { - try { - kerberosUser.logout(); - } catch (LoginException excp) { - LOG.error("Error logging out keytab user", excp); - } finally { - kerberosUser = null; - } - } - } - - synchronized void connect() { - SolrClient me = solrClient; - if (me == null) { - synchronized(SolrAuditDestination.class) { - me = solrClient; - if (solrClient == null) { - KeyManager[] kmList = getKeyManagers(); - TrustManager[] tmList = getTrustManagers(); - SSLContext sslContext = getSSLContext(kmList, tmList); - if(sslContext != null) { - SSLContext.setDefault(sslContext); - } - String urls = MiscUtil.getStringProperty(props, propPrefix - + "." + PROP_SOLR_URLS); - if (urls != null) { - urls = urls.trim(); - } - if (urls != null && urls.equalsIgnoreCase("NONE")) { - urls = null; - } - List solrURLs = new ArrayList(); - String zkHosts = null; - solrURLs = MiscUtil.toArray(urls, ","); - zkHosts = MiscUtil.getStringProperty(props, propPrefix + "." - + PROP_SOLR_ZK); - if (zkHosts != null && zkHosts.equalsIgnoreCase("NONE")) { - zkHosts = null; - } - String collectionName = MiscUtil.getStringProperty(props, - propPrefix + "." + PROP_SOLR_COLLECTION); - if (collectionName == null - || collectionName.equalsIgnoreCase("none")) { - collectionName = DEFAULT_COLLECTION_NAME; - } - - LOG.info("Solr zkHosts=" + zkHosts + ", solrURLs=" + urls - + ", collectionName=" + collectionName); - - if (zkHosts != null && !zkHosts.isEmpty()) { - LOG.info("Connecting to solr cloud using zkHosts=" - + zkHosts); - try { - // Instantiate - Krb5HttpClientBuilder krbBuild = new Krb5HttpClientBuilder(); - SolrHttpClientBuilder kb = krbBuild.getBuilder(); - HttpClientUtil.setHttpClientBuilder(kb); - - final List zkhosts = new ArrayList(Arrays.asList(zkHosts.split(","))); - final CloudSolrClient solrCloudClient = MiscUtil.executePrivilegedAction(new PrivilegedExceptionAction() { - @Override - public CloudSolrClient run() throws Exception { - CloudSolrClient solrCloudClient = new CloudSolrClient.Builder(zkhosts, Optional.empty()).build(); - return solrCloudClient; - }; - }); - - solrCloudClient.setDefaultCollection(collectionName); - me = solrClient = solrCloudClient; - } catch (Throwable t) { - LOG.fatal("Can't connect to Solr server. ZooKeepers=" - + zkHosts, t); - } - } else if (solrURLs != null && !solrURLs.isEmpty()) { - try { - LOG.info("Connecting to Solr using URLs=" + solrURLs); - Krb5HttpClientBuilder krbBuild = new Krb5HttpClientBuilder(); - SolrHttpClientBuilder kb = krbBuild.getBuilder(); - HttpClientUtil.setHttpClientBuilder(kb); - final List solrUrls = solrURLs; - final LBHttpSolrClient lbSolrClient = MiscUtil.executePrivilegedAction(new PrivilegedExceptionAction() { - @Override - public LBHttpSolrClient run() throws Exception { - LBHttpSolrClient.Builder builder = new LBHttpSolrClient.Builder(); - builder.withBaseSolrUrl(solrUrls.get(0)); - builder.withConnectionTimeout(1000); - LBHttpSolrClient lbSolrClient = builder.build(); - return lbSolrClient; - }; - }); - - for (int i = 1; i < solrURLs.size(); i++) { - lbSolrClient.addSolrServer(solrURLs.get(i)); - } - me = solrClient = lbSolrClient; - } catch (Throwable t) { - LOG.fatal("Can't connect to Solr server. URL=" - + solrURLs, t); - } - } - } - } - } - } - - @Override - public boolean log(Collection events) { - boolean ret = false; - try { - logStatusIfRequired(); - addTotalCount(events.size()); - - if (solrClient == null) { - connect(); - if (solrClient == null) { - // Solr is still not initialized. So need return error - addDeferredCount(events.size()); - return ret; - } - } - - final Collection docs = new ArrayList(); - for (AuditEventBase event : events) { - AuthzAuditEvent authzEvent = (AuthzAuditEvent) event; - // Convert AuditEventBase to Solr document - SolrInputDocument document = toSolrDoc(authzEvent); - docs.add(document); - } - try { - final UpdateResponse response = addDocsToSolr(solrClient, docs); - - if (response.getStatus() != 0) { - addFailedCount(events.size()); - logFailedEvent(events, response.toString()); - } else { - addSuccessCount(events.size()); - ret = true; - } - } catch (SolrException ex) { - addFailedCount(events.size()); - logFailedEvent(events, ex); - } - } catch (Throwable t) { - addDeferredCount(events.size()); - logError("Error sending message to Solr", t); - } - return ret; - } - - /* - * (non-Javadoc) - * - * @see org.apache.ranger.audit.provider.AuditProvider#flush() - */ - @Override - public void flush() { - - } - - SolrInputDocument toSolrDoc(AuthzAuditEvent auditEvent) { - SolrInputDocument doc = new SolrInputDocument(); - doc.addField("id", auditEvent.getEventId()); - doc.addField("access", auditEvent.getAccessType()); - doc.addField("enforcer", auditEvent.getAclEnforcer()); - doc.addField("agent", auditEvent.getAgentId()); - doc.addField("repo", auditEvent.getRepositoryName()); - doc.addField("sess", auditEvent.getSessionId()); - doc.addField("reqUser", auditEvent.getUser()); - doc.addField("reqData", auditEvent.getRequestData()); - doc.addField("resource", auditEvent.getResourcePath()); - doc.addField("cliIP", auditEvent.getClientIP()); - doc.addField("logType", auditEvent.getLogType()); - doc.addField("result", auditEvent.getAccessResult()); - doc.addField("policy", auditEvent.getPolicyId()); - doc.addField("repoType", auditEvent.getRepositoryType()); - doc.addField("resType", auditEvent.getResourceType()); - doc.addField("reason", auditEvent.getResultReason()); - doc.addField("action", auditEvent.getAction()); - doc.addField("evtTime", auditEvent.getEventTime()); - doc.addField("seq_num", auditEvent.getSeqNum()); - doc.setField("event_count", auditEvent.getEventCount()); - doc.setField("event_dur_ms", auditEvent.getEventDurationMS()); - doc.setField("tags", auditEvent.getTags()); - doc.setField("cluster", auditEvent.getClusterName()); - doc.setField("zoneName", auditEvent.getZoneName()); - doc.setField("agentHost", auditEvent.getAgentHostname()); - doc.setField("policyVersion", auditEvent.getPolicyVersion()); - - return doc; - } - - public boolean isAsync() { - return true; - } - - private void init() { - LOG.info("==>SolrAuditDestination.init()" ); - try { - // SolrJ requires "java.security.auth.login.config" property to be set to identify itself that it is kerberized. So using a dummy property for it - // Acutal solrclient JAAS configs are read from the ranger--audit.xml present in components conf folder and set by InMemoryJAASConfiguration - // Refer InMemoryJAASConfiguration doc for JAAS Configuration - String confFileName = System.getProperty(PROP_JAVA_SECURITY_AUTH_LOGIN_CONFIG); - LOG.info("In solrAuditDestination.init() : JAAS Configuration set as [" + confFileName + "]"); - if ( System.getProperty(PROP_JAVA_SECURITY_AUTH_LOGIN_CONFIG) == null ) { - if ( MiscUtil.getBooleanProperty(props, propPrefix + "." + PROP_SOLR_FORCE_USE_INMEMORY_JAAS_CONFIG,false) ) { - System.setProperty(PROP_JAVA_SECURITY_AUTH_LOGIN_CONFIG, "/dev/null"); - } else { - LOG.warn("No Client JAAS config present in solr audit config. Ranger Audit to Kerberized Solr will fail..."); - } - } - - LOG.info("Loading SolrClient JAAS config from Ranger audit config if present..."); - - InMemoryJAASConfiguration conf = InMemoryJAASConfiguration.init(props); - - KerberosUser kerberosUser = new KerberosJAASConfigUser("Client", conf); - - if (kerberosUser.getPrincipal() != null) { - this.kerberosUser = kerberosUser; - } - } catch (Exception e) { - LOG.error("ERROR: Unable to load SolrClient JAAS config from Audit config file. Audit to Kerberized Solr will fail...", e); - } finally { - String confFileName = System.getProperty(PROP_JAVA_SECURITY_AUTH_LOGIN_CONFIG); - LOG.info("In solrAuditDestination.init() (finally) : JAAS Configuration set as [" + confFileName + "]"); - } - LOG.info("<==SolrAuditDestination.init()" ); - } - - private KeyManager[] getKeyManagers() { - KeyManager[] kmList = null; - String credentialProviderPath = MiscUtil.getStringProperty(props, RANGER_POLICYMGR_CLIENT_KEY_FILE_CREDENTIAL); - String keyStoreAlias = RANGER_POLICYMGR_CLIENT_KEY_FILE_CREDENTIAL_ALIAS; - String keyStoreFile = MiscUtil.getStringProperty(props, RANGER_POLICYMGR_CLIENT_KEY_FILE); - String keyStoreFilepwd = MiscUtil.getCredentialString(credentialProviderPath, keyStoreAlias); - if (StringUtils.isNotEmpty(keyStoreFile) && StringUtils.isNotEmpty(keyStoreFilepwd)) { - InputStream in = null; - - try { - in = getFileInputStream(keyStoreFile); - - if (in != null) { - String keyStoreType = MiscUtil.getStringProperty(props, RANGER_POLICYMGR_CLIENT_KEY_FILE_TYPE); - keyStoreType = StringUtils.isNotEmpty(keyStoreType) ? keyStoreType : RANGER_POLICYMGR_CLIENT_KEY_FILE_TYPE_DEFAULT; - KeyStore keyStore = KeyStore.getInstance(keyStoreType); - - keyStore.load(in, keyStoreFilepwd.toCharArray()); - - KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance(RANGER_SSL_KEYMANAGER_ALGO_TYPE); - - keyManagerFactory.init(keyStore, keyStoreFilepwd.toCharArray()); - - kmList = keyManagerFactory.getKeyManagers(); - } else { - LOG.error("Unable to obtain keystore from file [" + keyStoreFile + "]"); - } - } catch (KeyStoreException e) { - LOG.error("Unable to obtain from KeyStore :" + e.getMessage(), e); - } catch (NoSuchAlgorithmException e) { - LOG.error("SSL algorithm is NOT available in the environment", e); - } catch (CertificateException e) { - LOG.error("Unable to obtain the requested certification ", e); - } catch (FileNotFoundException e) { - LOG.error("Unable to find the necessary SSL Keystore Files", e); - } catch (IOException e) { - LOG.error("Unable to read the necessary SSL Keystore Files", e); - } catch (UnrecoverableKeyException e) { - LOG.error("Unable to recover the key from keystore", e); - } finally { - close(in, keyStoreFile); - } - } - - return kmList; - } - - private TrustManager[] getTrustManagers() { - TrustManager[] tmList = null; - String credentialProviderPath = MiscUtil.getStringProperty(props, RANGER_POLICYMGR_TRUSTSTORE_FILE_CREDENTIAL); - String trustStoreAlias = RANGER_POLICYMGR_TRUSTSTORE_FILE_CREDENTIAL_ALIAS; - String trustStoreFile = MiscUtil.getStringProperty(props, RANGER_POLICYMGR_TRUSTSTORE_FILE); - String trustStoreFilepwd = MiscUtil.getCredentialString(credentialProviderPath, trustStoreAlias); - if (StringUtils.isNotEmpty(trustStoreFile) && StringUtils.isNotEmpty(trustStoreFilepwd)) { - InputStream in = null; - - try { - in = getFileInputStream(trustStoreFile); - - if (in != null) { - String trustStoreType = MiscUtil.getStringProperty(props, RANGER_POLICYMGR_TRUSTSTORE_FILE_TYPE); - trustStoreType = StringUtils.isNotEmpty(trustStoreType) ? trustStoreType : RANGER_POLICYMGR_TRUSTSTORE_FILE_TYPE_DEFAULT; - KeyStore trustStore = KeyStore.getInstance(trustStoreType); - - trustStore.load(in, trustStoreFilepwd.toCharArray()); - - TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(RANGER_SSL_TRUSTMANAGER_ALGO_TYPE); - - trustManagerFactory.init(trustStore); - - tmList = trustManagerFactory.getTrustManagers(); - } else { - LOG.error("Unable to obtain truststore from file [" + trustStoreFile + "]"); - } - } catch (KeyStoreException e) { - LOG.error("Unable to obtain from KeyStore", e); - } catch (NoSuchAlgorithmException e) { - LOG.error("SSL algorithm is NOT available in the environment :" + e.getMessage(), e); - } catch (CertificateException e) { - LOG.error("Unable to obtain the requested certification :" + e.getMessage(), e); - } catch (FileNotFoundException e) { - LOG.error("Unable to find the necessary SSL TrustStore File:" + trustStoreFile, e); - } catch (IOException e) { - LOG.error("Unable to read the necessary SSL TrustStore Files :" + trustStoreFile, e); - } finally { - close(in, trustStoreFile); - } - } - - return tmList; - } - - private SSLContext getSSLContext(KeyManager[] kmList, TrustManager[] tmList) { - SSLContext sslContext = null; - try { - sslContext = SSLContext.getInstance(RANGER_SSL_CONTEXT_ALGO_TYPE); - if (sslContext != null) { - sslContext.init(kmList, tmList, new SecureRandom()); - } - } catch (NoSuchAlgorithmException e) { - LOG.error("SSL algorithm is not available in the environment", e); - } catch (KeyManagementException e) { - LOG.error("Unable to initialise the SSLContext", e); - } - return sslContext; - } - - private UpdateResponse addDocsToSolr(final SolrClient solrClient, final Collection docs) throws Exception { - final UpdateResponse ret; - - try { - final PrivilegedExceptionAction action = () -> solrClient.add(docs); - - if (kerberosUser != null) { - // execute the privileged action as the given keytab user - final KerberosAction kerberosAction = new KerberosAction<>(kerberosUser, action, LOG); - - ret = (UpdateResponse) kerberosAction.execute(); - } else { - ret = action.run(); - } - } catch (Exception e) { - throw e; - } - - return ret; - } - - private InputStream getFileInputStream(String fileName) throws IOException { - InputStream in = null; - if (StringUtils.isNotEmpty(fileName)) { - File file = new File(fileName); - if (file != null && file.exists()) { - in = new FileInputStream(file); - } else { - in = ClassLoader.getSystemResourceAsStream(fileName); - } - } - return in; - } - - private void close(InputStream str, String filename) { - if (str != null) { - try { - str.close(); - } catch (IOException excp) { - LOG.error("Error while closing file: [" + filename + "]", excp); - } - } - } -} diff --git a/auth-audits/src/main/java/org/apache/atlas/audit/provider/AuditProviderFactory.java b/auth-audits/src/main/java/org/apache/atlas/audit/provider/AuditProviderFactory.java index 269da05423..7b67e9c640 100644 --- a/auth-audits/src/main/java/org/apache/atlas/audit/provider/AuditProviderFactory.java +++ b/auth-audits/src/main/java/org/apache/atlas/audit/provider/AuditProviderFactory.java @@ -23,7 +23,6 @@ import org.apache.hadoop.util.ShutdownHookManager; import org.apache.atlas.audit.destination.*; import org.apache.atlas.audit.provider.hdfs.HdfsAuditProvider; -import org.apache.atlas.audit.provider.solr.SolrAuditProvider; import org.apache.atlas.audit.queue.AuditAsyncQueue; import org.apache.atlas.audit.queue.AuditBatchQueue; import org.apache.atlas.audit.queue.AuditFileQueue; @@ -331,19 +330,6 @@ public synchronized void init(Properties props, String appType) { } }*/ - if (isAuditToSolrEnabled) { - LOG.info("SolrAuditProvider is enabled"); - SolrAuditProvider solrProvider = new SolrAuditProvider(); - solrProvider.init(props); - - if (solrProvider.isAsync()) { - AsyncAuditProvider asyncProvider = new AsyncAuditProvider( - "MySolrAuditProvider", 1000, 1000, solrProvider); - providers.add(asyncProvider); - } else { - providers.add(solrProvider); - } - } if (isAuditToLog4jEnabled) { Log4jAuditProvider log4jProvider = new Log4jAuditProvider(); @@ -416,8 +402,6 @@ private AuditHandler getProviderFromConfig(Properties props, provider = new FileAuditDestination(); } else if (providerName.equalsIgnoreCase("hdfs")) { provider = new HDFSAuditDestination(); - } else if (providerName.equalsIgnoreCase("solr")) { - provider = new SolrAuditDestination(); } else if (providerName.equalsIgnoreCase("elasticsearch")) { provider = new ElasticSearchAuditDestination(); } /*else if (providerName.equalsIgnoreCase("kafka")) { diff --git a/auth-audits/src/main/java/org/apache/atlas/audit/provider/solr/SolrAuditProvider.java b/auth-audits/src/main/java/org/apache/atlas/audit/provider/solr/SolrAuditProvider.java deleted file mode 100644 index 914f45685e..0000000000 --- a/auth-audits/src/main/java/org/apache/atlas/audit/provider/solr/SolrAuditProvider.java +++ /dev/null @@ -1,303 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.atlas.audit.provider.solr; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.atlas.audit.destination.AuditDestination; -import org.apache.atlas.audit.model.AuditEventBase; -import org.apache.atlas.audit.model.AuthzAuditEvent; -import org.apache.atlas.audit.provider.MiscUtil; -import org.apache.atlas.audit.utils.SolrAppUtil; -import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.impl.HttpSolrClient; -import org.apache.solr.client.solrj.response.UpdateResponse; -import org.apache.solr.common.SolrInputDocument; - -import java.io.IOException; -import java.security.PrivilegedExceptionAction; -import java.util.Collection; -import java.util.Collections; -import java.util.Date; -import java.util.Properties; - -public class SolrAuditProvider extends AuditDestination { - private static final Log LOG = LogFactory.getLog(SolrAuditProvider.class); - - public static final String AUDIT_MAX_QUEUE_SIZE_PROP = "xasecure.audit.solr.async.max.queue.size"; - public static final String AUDIT_MAX_FLUSH_INTERVAL_PROP = "xasecure.audit.solr.async.max.flush.interval.ms"; - public static final String AUDIT_RETRY_WAIT_PROP = "xasecure.audit.solr.retry.ms"; - - static final Object lock = new Object(); - volatile SolrClient solrClient = null; - Date lastConnectTime = null; - long lastFailTime = 0; - - int retryWaitTime = 30000; - - public SolrAuditProvider() { - } - - @Override - public void init(Properties props) { - LOG.info("init() called"); - super.init(props); - - retryWaitTime = MiscUtil.getIntProperty(props, - AUDIT_RETRY_WAIT_PROP, retryWaitTime); - } - - void connect() { - SolrClient me = solrClient; - if (me == null) { - synchronized (lock) { - me = solrClient; - if (me == null) { - final String solrURL = MiscUtil.getStringProperty(props, - "xasecure.audit.solr.solr_url"); - - if (lastConnectTime != null) { - // Let's wait for enough time before retrying - long diff = System.currentTimeMillis() - - lastConnectTime.getTime(); - if (diff < retryWaitTime) { - if (LOG.isDebugEnabled()) { - LOG.debug("Ignore connecting to solr url=" - + solrURL + ", lastConnect=" + diff - + "ms"); - } - return; - } - } - lastConnectTime = new Date(); - - if (solrURL == null || solrURL.isEmpty()) { - LOG.fatal("Solr URL for Audit is empty"); - return; - } - - try { - // TODO: Need to support SolrCloud also - solrClient = MiscUtil.executePrivilegedAction(new PrivilegedExceptionAction() { - @Override - public SolrClient run() throws Exception { - HttpSolrClient.Builder builder = new HttpSolrClient.Builder(); - builder.withBaseSolrUrl(solrURL); - builder.allowCompression(true); - builder.withConnectionTimeout(1000); - HttpSolrClient httpSolrClient = builder.build(); - return httpSolrClient; - }; - }); - - me = solrClient; - } catch (Throwable t) { - LOG.fatal("Can't connect to Solr server. URL=" - + solrURL, t); - } - } - } - } - } - - /* - * (non-Javadoc) - * - * @see - * org.apache.ranger.audit.provider.AuditProvider#log(org.apache.ranger. - * audit.model.AuditEventBase) - */ - @Override - public boolean log(AuditEventBase event) { - if (!(event instanceof AuthzAuditEvent)) { - LOG.error(event.getClass().getName() - + " audit event class type is not supported"); - return false; - } - AuthzAuditEvent authzEvent = (AuthzAuditEvent) event; - // TODO: This should be done at a higher level - - if (authzEvent.getAgentHostname() == null) { - authzEvent.setAgentHostname(MiscUtil.getHostname()); - } - - if (authzEvent.getLogType() == null) { - authzEvent.setLogType("RangerAudit"); - } - - if (authzEvent.getEventId() == null) { - authzEvent.setEventId(MiscUtil.generateUniqueId()); - } - - try { - if (solrClient == null) { - connect(); - if (solrClient == null) { - // Solr is still not initialized. So need to throw error - return false; - } - } - - if (lastFailTime > 0) { - long diff = System.currentTimeMillis() - lastFailTime; - if (diff < retryWaitTime) { - if (LOG.isDebugEnabled()) { - LOG.debug("Ignore sending audit. lastConnect=" + diff - + " ms"); - } - return false; - } - } - // Convert AuditEventBase to Solr document - final SolrInputDocument document = toSolrDoc(authzEvent); - final Collection docs = Collections.singletonList(document); - final UpdateResponse response = SolrAppUtil.addDocsToSolr(solrClient, docs); - - if (response.getStatus() != 0) { - lastFailTime = System.currentTimeMillis(); - - // System.out.println("Response=" + response.toString() - // + ", status= " + response.getStatus() + ", event=" - // + event); - // throw new Exception("Aborting. event=" + event + - // ", response=" - // + response.toString()); - } else { - lastFailTime = 0; - } - - } catch (Throwable t) { - LOG.error("Error sending message to Solr", t); - return false; - } - return true; - } - - @Override - public boolean log(Collection events) { - for (AuditEventBase event : events) { - log(event); - } - return true; - } - - @Override - public boolean logJSON(String event) { - AuditEventBase eventObj = MiscUtil.fromJson(event, - AuthzAuditEvent.class); - return log(eventObj); - } - - @Override - public boolean logJSON(Collection events) { - for (String event : events) { - logJSON(event); - } - return false; - } - - /* - * (non-Javadoc) - * - * @see org.apache.ranger.audit.provider.AuditProvider#start() - */ - @Override - public void start() { - connect(); - } - - /* - * (non-Javadoc) - * - * @see org.apache.ranger.audit.provider.AuditProvider#stop() - */ - @Override - public void stop() { - LOG.info("SolrAuditProvider.stop() called.."); - try { - if (solrClient != null) { - solrClient.close(); - } - } catch (IOException ioe) { - LOG.error("Error while stopping slor!", ioe); - } finally { - solrClient = null; - } - } - - /* - * (non-Javadoc) - * - * @see org.apache.ranger.audit.provider.AuditProvider#waitToComplete() - */ - @Override - public void waitToComplete() { - - } - - - @Override - public void waitToComplete(long timeout) { - - } - - /* - * (non-Javadoc) - * - * @see org.apache.ranger.audit.provider.AuditProvider#flush() - */ - @Override - public void flush() { - // TODO Auto-generated method stub - - } - - SolrInputDocument toSolrDoc(AuthzAuditEvent auditEvent) { - SolrInputDocument doc = new SolrInputDocument(); - doc.addField("id", auditEvent.getEventId()); - doc.addField("access", auditEvent.getAccessType()); - doc.addField("enforcer", auditEvent.getAclEnforcer()); - doc.addField("agent", auditEvent.getAgentId()); - doc.addField("repo", auditEvent.getRepositoryName()); - doc.addField("sess", auditEvent.getSessionId()); - doc.addField("reqUser", auditEvent.getUser()); - doc.addField("reqData", auditEvent.getRequestData()); - doc.addField("resource", auditEvent.getResourcePath()); - doc.addField("cliIP", auditEvent.getClientIP()); - doc.addField("logType", auditEvent.getLogType()); - doc.addField("result", auditEvent.getAccessResult()); - doc.addField("policy", auditEvent.getPolicyId()); - doc.addField("repoType", auditEvent.getRepositoryType()); - doc.addField("resType", auditEvent.getResourceType()); - doc.addField("reason", auditEvent.getResultReason()); - doc.addField("action", auditEvent.getAction()); - doc.addField("evtTime", auditEvent.getEventTime()); - doc.addField("tags", auditEvent.getTags()); - doc.addField("cluster", auditEvent.getClusterName()); - doc.addField("zone", auditEvent.getZoneName()); - doc.addField("agentHost", auditEvent.getAgentHostname()); - return doc; - } - - public boolean isAsync() { - return true; - } - -} diff --git a/auth-audits/src/main/java/org/apache/atlas/audit/utils/SolrAppUtil.java b/auth-audits/src/main/java/org/apache/atlas/audit/utils/SolrAppUtil.java deleted file mode 100644 index c232b945fc..0000000000 --- a/auth-audits/src/main/java/org/apache/atlas/audit/utils/SolrAppUtil.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.atlas.audit.utils; - -import org.apache.atlas.audit.provider.MiscUtil; -import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.response.UpdateResponse; -import org.apache.solr.common.SolrInputDocument; - -import java.security.PrivilegedExceptionAction; -import java.util.Collection; - -public class SolrAppUtil { - public static UpdateResponse addDocsToSolr(final SolrClient solrClient, final Collection docs) throws Exception { - return MiscUtil.executePrivilegedAction(new PrivilegedExceptionAction() { - @Override - public UpdateResponse run() throws Exception { - return solrClient.add(docs); - } - }); - } -} diff --git a/graphdb/janus/pom.xml b/graphdb/janus/pom.xml index 7ae163ec5e..3662d6b3b7 100644 --- a/graphdb/janus/pom.xml +++ b/graphdb/janus/pom.xml @@ -203,22 +203,6 @@ - - - org.apache.solr - solr-solrj - ${solr.version} - - - org.codehaus.woodstox - woodstox-core-asl - - - io.netty - * - - - org.apache.lucene diff --git a/notification/pom.xml b/notification/pom.xml index 4d7c810708..3cd4e07de9 100644 --- a/notification/pom.xml +++ b/notification/pom.xml @@ -151,11 +151,6 @@ hadoop-common ${hadoop.version} - - org.apache.hadoop - hadoop-auth - ${hadoop.version} - org.slf4j slf4j-api diff --git a/pom.xml b/pom.xml index 6e23472641..92764dc646 100644 --- a/pom.xml +++ b/pom.xml @@ -709,7 +709,7 @@ 29.0-jre 4.1.0 ${hadoop.version} - 3.3.0 + 3.3.6 2.3.3 3.1.0 0.8.1 @@ -1041,12 +1041,6 @@ - - org.apache.hadoop - hadoop-auth - ${hadoop.version} - - org.apache.hadoop hadoop-client diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java index 3a730b0ce9..5c3410927e 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java @@ -128,9 +128,7 @@ public GraphBackedSearchIndexer(AtlasTypeRegistry typeRegistry) throws AtlasExce throws IndexException, RepositoryException { this.provider = provider; this.typeRegistry = typeRegistry; - - //make sure solr index follows graph backed index listener - addIndexListener(new SolrIndexHelper(typeRegistry)); + if (!HAConfiguration.isHAEnabled(configuration)) { initialize(provider.get());